Upgrade

.mvn/wrapper/maven-wrapper.properties

@@ -14,5 +14,5 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.6/apache-maven-3.8.6-bin.zip
-wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.0/apache-maven-3.9.0-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.1/maven-wrapper-3.1.1.jar

demo/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>org.springframework.boot</groupId>
 		<artifactId>spring-boot-starter-parent</artifactId>
-		<version>3.0.3</version>
+		<version>3.0.4</version>
 		<relativePath />
 	</parent>
 	<groupId>ch.rasc.hibppasswords</groupId>

importer/pom.xml

@@ -6,7 +6,7 @@
 	<groupId>ch.rasc.hibppasswords</groupId>
 	<artifactId>importer</artifactId>
 	<packaging>jar</packaging>
-	<version>1.0.0</version>
+	<version>1.1.0</version>
 
 	<description>Have I Been Pwned passwords Xodus database importer</description>
 	<inceptionYear>2019</inceptionYear>
@@ -101,7 +101,6 @@
 					<header>${basedir}/../config/header.txt</header>
 					<properties>
 						<year>${project.inceptionYear}</year>
-						<currentYear>2019</currentYear>
 					</properties>
 					<excludes>
 						<exclude>**/*.xml</exclude>

importer/src/main/java/ch/rasc/hibppasswords/importer/HibpPasswordsImporter.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019-2019 the original author or authors.
+ * Copyright the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,6 +19,8 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.jetbrains.annotations.NotNull;
@@ -39,10 +41,10 @@ public static void main(String[] args) throws Exception {
 
 		if (args.length == 3) {
 			if (args[0].equalsIgnoreCase("import")) {
-				Path hibpPasswordsFile = Paths.get(args[1]);
+				Path hibpHashesDirectory = Paths.get(args[1]);
 				Path databaseDir = Paths.get(args[2]);
 
-				int exitCode = HibpPasswordsImporter.doImport(hibpPasswordsFile,
+				int exitCode = HibpPasswordsImporter.doImport(hibpHashesDirectory,
 						databaseDir);
 				System.exit(exitCode);
 			}
@@ -75,18 +77,18 @@ else if (args[0].equalsIgnoreCase("query-sha1")) {
 
 	private static void printUsage() {
 		System.out.println(
-				"java -jar hibp-passwords-importer.jar import <hibp passwords txt file>  <database directory>");
+				"java -jar hibp-passwords-importer.jar import <hibp hashes directory>  <database directory>");
 		System.out.println(
 				"java -jar hibp-passwords-importer.jar query-plain <plain text password>  <database directory>");
 		System.out.println(
 				"java -jar hibp-passwords-importer.jar query-sha1 <sha1>  <database directory>");
 	}
 
-	private static int doImport(Path hibpPasswordsFile, Path databaseDir)
+	private static int doImport(Path hibpHashesDirectory, Path databaseDir)
 			throws Exception {
 
-		if (!Files.exists(hibpPasswordsFile)) {
-			System.out.println("hibp passwords text file does not exist");
+		if (!Files.exists(hibpHashesDirectory)) {
+			System.out.println("hibp directory does not exist");
 			return 1;
 		}
 
@@ -98,15 +100,31 @@ private static int doImport(Path hibpPasswordsFile, Path databaseDir)
 				Store store = env.openStore("passwords",
 						StoreConfig.WITHOUT_DUPLICATES_WITH_PREFIXING, txn);
 				try {
-					AtomicLong counter = new AtomicLong();
-					Files.lines(hibpPasswordsFile).forEach(line -> {
-						long c = counter.incrementAndGet();
-						if (c % 10_000_000 == 0) {
-							System.out.printf("imported: %d \n", c);
-							txn.flush();
+
+
+			        final AtomicLong importCounter = new AtomicLong(0L);
+			        final AtomicLong fileCounter = new AtomicLong(0L);
+
+					List<String> hashFiles = listAllFiles(hibpHashesDirectory);
+					int totalFiles = hashFiles.size();
+					for (String hashFile : hashFiles) {
+						Path inputFile = Paths.get(hashFile);
+						try (var linesReader = Files.lines(inputFile)) {
+							linesReader.forEach(line -> {
+								long c = importCounter.incrementAndGet();
+					              if (c > 10_000_000) {
+					                txn.flush();
+					                System.out.println(
+					                    "Processed no of files " + fileCounter.get() + " of " + totalFiles);
+					                importCounter.set(0L);
+					              }
+					              String hashPrefix = hashFile.substring(0, hashFile.lastIndexOf("."));
+								importLine(store, txn, hashPrefix, line);
+							});
 						}
-						importLine(store, txn, line);
-					});
+
+						fileCounter.incrementAndGet();
+					}
 
 					txn.commit();
 				}
@@ -119,12 +137,12 @@ private static int doImport(Path hibpPasswordsFile, Path databaseDir)
 		}
 	}
 
-	private static void importLine(Store store, Transaction txn, String line) {
-		String sha1 = line.substring(0, 40);
-		int count = Integer.parseInt(line.substring(41).trim());
+	private static void importLine(Store store, Transaction txn, String prefix, String line) {
+	    String sha1 = line.substring(0, 35);
+	    int count = Integer.parseInt(line.substring(36).trim());
 
-		ByteIterable key = new ArrayByteIterable(hexStringToByteArray(sha1));
-		store.putRight(txn, key, IntegerBinding.intToCompressedEntry(count));
+	    ByteIterable key = new ArrayByteIterable(hexStringToByteArray(prefix + sha1));
+	    store.putRight(txn, key, IntegerBinding.intToCompressedEntry(count));
 	}
 
 	private static byte[] hexStringToByteArray(String s) {
@@ -136,4 +154,21 @@ private static byte[] hexStringToByteArray(String s) {
 		return data;
 	}
 
+	private static List<String> listAllFiles(Path inputDir) {
+		List<String> files = new ArrayList<>();
+		try (var walker = Files.walk(inputDir)) {
+			walker.forEach(filePath -> {
+				if (Files.isRegularFile(filePath)) {
+					files.add(filePath.toString());
+				}
+			});
+		}
+		catch (IOException e) {
+			e.printStackTrace();
+		}
+
+		files.sort(String::compareTo);
+		return files;
+	}
+
 }

query/pom.xml

@@ -130,7 +130,6 @@
 					<header>${basedir}/../config/header.txt</header>
 					<properties>
 						<year>${project.inceptionYear}</year>
-						<currentYear>2019</currentYear>
 					</properties>
 					<excludes>
 						<exclude>**/*.xml</exclude>

query/src/main/java/ch/rasc/hibppasswords/query/HibpPasswordsQuery.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019-2019 the original author or authors.
+ * Copyright the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

query/src/main/java/ch/rasc/hibppasswords/query/RangeQueryResult.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019-2019 the original author or authors.
+ * Copyright the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.