Add an anonymous ID to users

Any new users will have an ID automatically assigned. Anonymous IDs will
be assigned to current users when they're pulled from the DB for the
first time after upgrading the service.

Next steps:
* Expose in /me and testmode and admin /user endpoints
* Add admin method to look up users by anonymous ID
* Release notes

build.xml

@@ -389,6 +389,7 @@
         <classpath refid="test.classpath"/>
         <formatter type="plain" usefile="false" />
         <sysproperty key="AUTH2_TEST_CONFIG" value="${testcfg}"/>
+        <test name="us.kbase.test.auth2.lib.storage.mongo.MongoStorageAnonymousIDBackfillingTest"/>
         <test name="us.kbase.test.auth2.lib.storage.mongo.MongoStorageConfigTest"/>
         <test name="us.kbase.test.auth2.lib.storage.mongo.MongoStorageCustomRoleTest"/>
         <test name="us.kbase.test.auth2.lib.storage.mongo.MongoStorageTestRoleTest"/>

src/us/kbase/auth2/lib/Authentication.java

@@ -394,7 +394,7 @@ public void createRoot(final Password pwd)
 				throw new RuntimeException("This is impossible", e);
 			}
 			final LocalUser root = LocalUser.getLocalUserBuilder(
-					UserName.ROOT, dn, clock.instant()).build();
+					UserName.ROOT, randGen.randomUUID(), dn, clock.instant()).build();
 			try {
 				storage.createLocalUser(root, new PasswordHashAndSalt(passwordHash, salt));
 				// only way to avoid a race condition. Checking existence before creating user
@@ -462,7 +462,7 @@ public Password createLocalUser(
 			pwd_copy = pwd.getPassword();
 			passwordHash = pwdcrypt.getEncryptedPassword(pwd_copy, salt);
 			final LocalUser lu = LocalUser.getLocalUserBuilder(
-					userName, displayName, clock.instant())
+					userName, randGen.randomUUID(), displayName, clock.instant())
 					.withEmailAddress(email).withForceReset(true).build();
 			storage.createLocalUser(lu, new PasswordHashAndSalt(passwordHash, salt));
 			logInfo("Local user {} created by admin {}",
@@ -1954,7 +1954,8 @@ public NewToken createUser(
 					"Not authorized to create user with remote identity %s", identityID));
 		}
 		final Instant now = clock.instant();
-		final NewUser.Builder b = NewUser.getBuilder(userName, displayName, now, match.get())
+		final NewUser.Builder b = NewUser.getBuilder(
+				userName, randGen.randomUUID(), displayName, now, match.get())
 				 // no need to set last login, will be set in the login() call below
 				.withEmailAddress(email);
 		for (final PolicyID pid: policyIDs) {
@@ -2054,7 +2055,11 @@ public void testModeCreateUser(final UserName userName, final DisplayName displa
 		}
 		final Instant now = clock.instant();
 		storage.testModeCreateUser(
-				userName, displayName, now, now.plusMillis(TEST_MODE_DATA_LIFETIME_MS));
+				userName,
+				randGen.randomUUID(),
+				displayName,
+				now,
+				now.plusMillis(TEST_MODE_DATA_LIFETIME_MS));
 		logInfo("Created test mode user {}", userName.getName());
 	}
 
@@ -3146,7 +3151,8 @@ public void importUser(final UserName userName, final RemoteIdentity remoteIdent
 			email = EmailAddress.UNKNOWN;
 		}
 		try {
-			storage.createUser(NewUser.getBuilder(userName, dn, clock.instant(), remoteIdentity)
+			storage.createUser(NewUser.getBuilder(
+					userName, randGen.randomUUID(), dn, clock.instant(), remoteIdentity)
 					.withEmailAddress(email).build());
 			logInfo("Imported user {}", userName.getName());
 		} catch (NoSuchRoleException e) {

src/us/kbase/auth2/lib/storage/AuthStorage.java

@@ -119,14 +119,17 @@ void createUser(NewUser newUser)
 	 * Note that {@link AuthUser#isLocal()} returns true for test users since local users are
 	 * defined as having no remote identities, but test users still have no passwords.
 	 * @param name the user's name.
+	 * @param anonymousID the anonymous ID for the user. The calling code is responsible for
+	 * ensuring these IDs are unique per user.
 	 * @param display the user's display name.
 	 * @param created the date the user was created.
 	 * @param expires the date the user expires from the system.
 	 * @throws UserExistsException if the user already exists.
 	 * @throws AuthStorageException if a problem connecting with the storage
 	 * system occurs. 
 	 */
-	void testModeCreateUser(UserName name, DisplayName display, Instant created, Instant expires)
+	void testModeCreateUser(
+			UserName name, UUID anonymousID, DisplayName display, Instant created, Instant expires)
 			throws UserExistsException, AuthStorageException;
 
 	/** Disable a user account.

src/us/kbase/auth2/lib/storage/mongo/Fields.java

@@ -20,6 +20,8 @@ public class Fields {
 
 	/** The user name. */
 	public static final String USER_NAME = "user";
+	/** The anonymous ID for the user. */
+	public static final String USER_ANONYMOUS_ID = "anonid";
 	/** The display name for the user. */
 	public static final String USER_DISPLAY_NAME = "display";
 	/** The canonical version of the display name. E.g. split into parts, whitespace removed, etc.

src/us/kbase/auth2/lib/storage/mongo/MongoStorage.java

@@ -6,6 +6,7 @@
 
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.security.NoSuchAlgorithmException;
 import java.time.Clock;
 import java.time.Instant;
 import java.util.Arrays;
@@ -46,6 +47,8 @@
 import com.mongodb.client.result.DeleteResult;
 import com.mongodb.client.result.UpdateResult;
 
+import us.kbase.auth2.cryptutils.RandomDataGenerator;
+import us.kbase.auth2.cryptutils.SHA1RandomDataGenerator;
 import us.kbase.auth2.lib.CustomRole;
 import us.kbase.auth2.lib.DisplayName;
 import us.kbase.auth2.lib.EmailAddress;
@@ -285,26 +288,40 @@ public class MongoStorage implements AuthStorage {
 
 	private final MongoDatabase db;
 	private final Clock clock;
+	private RandomDataGenerator randGen;
 
 	/** Create a new MongoDB authentication storage system.
 	 * @param db the MongoDB database to use for storage.
 	 * @throws StorageInitException if the storage system could not be initialized.
 	 */
 	public MongoStorage(final MongoDatabase db) throws StorageInitException {
-		this(db, Clock.systemDefaultZone()); //don't use timezone
+		this(db, getDefaultRandomGenerator(), Clock.systemDefaultZone()); //don't use timezone
 	}
 
 	// this should only be used for tests
-	private MongoStorage(final MongoDatabase db, final Clock clock) throws StorageInitException {
+	private MongoStorage(
+			final MongoDatabase db,
+			final RandomDataGenerator randGen,
+			final Clock clock)
+			throws StorageInitException {
 		requireNonNull(db, "db");
 		this.db = db;
+		this.randGen = randGen;
 		this.clock = clock;
 
 		//TODO MISC port over schemamanager from UJS (will need changes for schema key & mdb ver)
 		ensureIndexes(); // MUST come before checkConfig();
 		checkConfig();
 	}
 
+	private static RandomDataGenerator getDefaultRandomGenerator() {
+		try {
+			return new SHA1RandomDataGenerator();
+		} catch (NoSuchAlgorithmException e) {
+			throw new RuntimeException("This should be impossible", e);
+		}
+	}
+
 	private void checkConfig() throws StorageInitException  {
 		final MongoCollection<Document> col = db.getCollection(COL_CONFIG);
 		final Document cfg = new Document(Fields.DB_SCHEMA_KEY, Fields.DB_SCHEMA_VALUE);
@@ -386,6 +403,7 @@ public void createLocalUser(final LocalUser local, final PasswordHashAndSalt cre
 		final Optional<Instant> reset = local.getLastPwdReset();
 		final Document u = new Document(
 				Fields.USER_NAME, local.getUserName().getName())
+				.append(Fields.USER_ANONYMOUS_ID, local.getAnonymousID().toString())
 				.append(Fields.USER_LOCAL, true)
 				.append(Fields.USER_EMAIL, local.getEmail().getAddress())
 				.append(Fields.USER_DISPLAY_NAME, local.getDisplayName().getName())
@@ -432,6 +450,7 @@ public LocalUser getLocalUser(final UserName userName)
 
 		final LocalUser.Builder b = LocalUser.getLocalUserBuilder(
 				getUserName(user.getString(Fields.USER_NAME)),
+				UUID.fromString(user.getString(Fields.USER_ANONYMOUS_ID)),
 				getDisplayName(user.getString(Fields.USER_DISPLAY_NAME)),
 				user.getDate(Fields.USER_CREATED).toInstant())
 				.withEmailAddress(getEmail(user.getString(Fields.USER_EMAIL)))
@@ -626,6 +645,7 @@ public void createUser(final NewUser newUser)
 				COL_CUST_ROLES, newUser.getCustomRoles()).values();
 		final Document u = new Document(
 				Fields.USER_NAME, newUser.getUserName().getName())
+				.append(Fields.USER_ANONYMOUS_ID, newUser.getAnonymousID().toString())
 				.append(Fields.USER_LOCAL, false)
 				.append(Fields.USER_EMAIL, newUser.getEmail().getAddress())
 				.append(Fields.USER_DISPLAY_NAME, newUser.getDisplayName().getName())
@@ -667,11 +687,13 @@ public void createUser(final NewUser newUser)
 	@Override
 	public void testModeCreateUser(
 			final UserName name,
+			final UUID anonymousID,
 			final DisplayName display,
 			final Instant created,
 			final Instant expires)
 			throws UserExistsException, AuthStorageException {
 		requireNonNull(name, "name");
+		requireNonNull(anonymousID, "anonymousID");
 		requireNonNull(display, "display");
 		requireNonNull(created, "created");
 		requireNonNull(expires, "expires");
@@ -680,6 +702,7 @@ public void testModeCreateUser(
 		}
 		final Document u = new Document(
 				Fields.USER_NAME, name.getName())
+				.append(Fields.USER_ANONYMOUS_ID, anonymousID.toString())
 				.append(Fields.USER_LOCAL, false)
 				.append(Fields.USER_EMAIL, null)
 				.append(Fields.USER_DISPLAY_NAME, display.getName())
@@ -721,19 +744,54 @@ private Document getUserDoc(
 			final boolean local)
 			throws AuthStorageException, NoSuchUserException {
 		requireNonNull(userName, "userName");
-		final Document projection = new Document(Fields.USER_PWD_HSH, 0)
-				.append(Fields.USER_SALT, 0);
-		final Document user = findOne(collection,
-				new Document(Fields.USER_NAME, userName.getName()), projection);
+		Document user = getUserDocWithoutPassword(collection, userName);
 		if (user == null) {
 			throw new NoSuchUserException(userName.getName());
 		}
+		if (user.getString(Fields.USER_ANONYMOUS_ID) == null) {
+			user = updateUserAnonID(collection, userName);
+		}
 		if (local && !user.getBoolean(Fields.USER_LOCAL)) {
 			throw new NoSuchLocalUserException(userName.getName());
 		}
 		return user;
 	}
 
+	private Document getUserDocWithoutPassword(final String collection, final UserName userName)
+			throws AuthStorageException {
+		return findOne(
+				collection,
+				new Document(Fields.USER_NAME, userName.getName()),
+				new Document(Fields.USER_PWD_HSH, 0).append(Fields.USER_SALT, 0));
+	}
+
+	private Document updateUserAnonID(final String collection, final UserName userName)
+			throws AuthStorageException {
+		/* Added in version 0.6.0 when anonymous IDs were added to users. This method lazily
+		 * backfills the anonymous ID on an as-needed basis.
+		 */
+		final Document query = new Document(Fields.USER_NAME, userName.getName())
+				// only modify if not already done to avoid race conditions
+				.append(Fields.USER_ANONYMOUS_ID, null);
+		final Document update = new Document(
+				"$set", new Document(Fields.USER_ANONYMOUS_ID, randGen.randomUUID().toString()));
+		try {
+			db.getCollection(COL_USERS).updateOne(query, update);
+			// if it didn't match we assume the user doc was updated to include an ID in
+			// a different thread or process
+		} catch (MongoException e) { // this is very difficult to test
+			throw new AuthStorageException(
+					"Connection to database failed: " + e.getMessage(), e);
+		}
+		// pull the user from the DB again to avoid a race condition here
+		final Document userdoc = getUserDocWithoutPassword(collection, userName);
+		if (userdoc == null) {
+			throw new RuntimeException(
+					"User unexpectedly not found in database: " + userName.getName());
+		}
+		return userdoc;
+	}
+
 	@Override
 	public void disableAccount(final UserName user, final UserName admin, final String reason)
 			throws NoSuchUserException, AuthStorageException {
@@ -968,6 +1026,7 @@ private AuthUser toUser(final Document user, final boolean testUser)
 
 		final AuthUser.Builder b = AuthUser.getBuilder(
 				getUserName(user.getString(Fields.USER_NAME)),
+				UUID.fromString(user.getString(Fields.USER_ANONYMOUS_ID)),
 				getDisplayName(user.getString(Fields.USER_DISPLAY_NAME)),
 				user.getDate(Fields.USER_CREATED).toInstant())
 				.withEmailAddress(getEmail(user.getString(Fields.USER_EMAIL)))