CLC v1.9.5

Custom sql migration paths.
Allow to specify sql:postgres=file option
Add DSL compiler timings for creating the migration or source

When output files contain the specified extension, assume all of them have the correct extension.

CommandLineClient/pom.xml

@@ -4,7 +4,7 @@
 	<groupId>com.dslplatform</groupId>
 	<artifactId>dsl-clc</artifactId>
 	<packaging>jar</packaging>
-	<version>1.9.4</version>
+	<version>1.9.5</version>
 	<name>DSL Platform - Compiler Command-Line Client</name>
 	<url>https://github.com/ngs-doo/dsl-compiler-client</url>
 	<description>Command line client for interaction with DSL Platform compiler (https://dsl-platform.com)</description>

CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/ApplyMigration.java

@@ -133,7 +133,10 @@ private static void applyMigrationScript(final Context context, final File file,
 		}
 		context.show("Applying migration...");
 		db.execute(context, sql);
-		if (file.renameTo(new File(file.getParentFile(), "applied-" + file.getName()))) {
+		final String customFile = context.get("sql:" + db.getDName().toLowerCase());
+		if (customFile != null && file.getName().equals(customFile)) {
+			context.show("Database migrated via: " + file.getAbsolutePath());
+		} else if (file.renameTo(new File(file.getParentFile(), "applied-" + file.getName()))) {
 			context.show("Database migrated and script renamed to: applied-" + file.getName());
 		} else {
 			context.show("Database migrated, but unable to rename script: " + file.getName());

CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/DslCompiler.java

@@ -56,11 +56,15 @@ public static Map<String, String> compile(
 			arguments.add("dsl=" + f.getAbsolutePath());
 		}
 		context.log("Compiling DSL to " + target + "...");
+		final long start = new Date().getTime();
 		final Either<byte[]> response = runCompiler(context, arguments);
 		if (!response.isSuccess()) {
 			context.error(response.whyNot());
 			throw new ExitException();
 		}
+		final long end = new Date().getTime();
+		context.show("Creating the source took " + (end - start) / 1000 + " second(s)");
+
 		final Either<Document> xml = Utils.readXml(new ByteArrayInputStream(response.get()));
 		if (!xml.isSuccess()) {
 			context.error(new String(response.get(), UTF_8));
@@ -635,7 +639,7 @@ public static File lookupDefaultPath(final Context context) throws ExitException
 		final String dslName = Download.isDefaultUrl(context) ? ".DSL-Platform" : ".DSL-Custom";
 		final File compilerFolder = new File(homePath, dslName);
 		if (!compilerFolder.exists() && !compilerFolder.mkdirs()) {
-			context.warning("Error creating dsl compiler path in: " + compilerFolder.getAbsolutePath() + ". Will use temporary filter instead.");
+			context.warning("Error creating dsl compiler path in: " + compilerFolder.getAbsolutePath() + ". Will use temporary folder instead.");
 			return new File(TempPath.getTempRootPath(context), "dsl-compiler.exe");
 		}
 		return new File(compilerFolder, "dsl-compiler.exe");

CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/DslPath.java

@@ -43,7 +43,7 @@ private static void findDsls(final Context context) throws ExitException {
 		String value = context.get(INSTANCE);
 		if (value == null) {
 			if (!(new File("./dsl").exists())) {
-				context.error("DSL path not provided. Can't use default path (./dsl) since it doesn't exists");
+				context.error("DSL path not provided. Can't use default path (./dsl) since it doesn't exist");
 				throw new ExitException();
 			}
 			context.put(INSTANCE, value = "./dsl");
@@ -88,15 +88,15 @@ public boolean check(final Context context) {
 		if (value == null) {
 			final File dslPath = new File("./dsl");
 			if (!dslPath.exists()) {
-				context.error("DSL path not provided. Can't use default path (./dsl) since it doesn't exists");
+				context.error("DSL path not provided. Can't use default path (./dsl) since it doesn't exist");
 				return false;
 			}
 			context.put(INSTANCE, "./dsl");
 		} else {
 			for (final String part : value.split(File.pathSeparator)) {
 				final File dslPath = new File(part).getAbsoluteFile();
 				if (!dslPath.exists()) {
-					context.error("Provided DSL path (" + part + ") does not exists. Please provide valid path to DSL files");
+					context.error("Provided DSL path (" + part + ") does not exist. Please provide valid path to DSL files");
 					return false;
 				}
 			}

CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/Migration.java

@@ -6,7 +6,7 @@
 import java.io.IOException;
 import java.util.*;
 
-public enum Migration implements CompileParameter {
+public enum Migration implements CompileParameter, ParameterParser {
 	INSTANCE;
 
 	@Override
@@ -33,7 +33,7 @@ public static File getOracleMigrationFile(final Context context) {
 		return context.load(ORACLE_MIGRATION_FILE_NAME);
 	}
 
-	public static String[] extractDescriptions(final String sql) throws ExitException {
+	public static String[] extractDescriptions(final String sql) {
 		final int start = sql.indexOf(DESCRIPTION_START);
 		final int end = sql.indexOf(DESCRIPTION_END);
 		if (end > start) {
@@ -42,11 +42,53 @@ public static String[] extractDescriptions(final String sql) throws ExitExceptio
 		return new String[0];
 	}
 
+	@Override
+	public Either<Boolean> tryParse(final String name, final String value, final Context context) {
+		if ("migration".equals(name)) {
+			context.put(name, value == null || value.length() == 0 ? null : value);
+			return Either.success(true);
+		} else {
+			for (final String db : new String[]{"postgres", "oracle"}) {
+				if (("sql:" + db).equalsIgnoreCase(name)) {
+					if (value == null || value.length() == 0) {
+						return Either.fail("Custom output file parameter detected, but it's missing file name as an argument. Parameter: " + name);
+					}
+					final File path = new File(value);
+					if (path.exists() && path.isDirectory()) {
+						return Either.fail("Output path found, but it's a directory. Parameter: " + name);
+					}
+					context.put("sql:" + db, value);
+					return Either.success(true);
+				} else if (name.startsWith("previous-sql:" + db)) {
+					if (value == null || value.length() == 0) {
+						return Either.fail("Previous sql file parameter detected, but it's missing path as an argument. Parameter: " + name);
+					}
+					final File previous = new File(value);
+					if (!previous.exists()) {
+						return Either.fail("Previous sql path provided, but file does not exist at: " + previous.getAbsolutePath() + ". Parameter: " + name);
+					} else if (previous.isDirectory()) {
+						return Either.fail("Previous sql path found, but it's a directory: " + previous.getAbsolutePath() + ". Parameter: " + name);
+					}
+					final Either<String> content = Utils.readFile(previous);
+					if (!content.isSuccess()) {
+						return Either.fail("Unable to read previous sql file from: " + previous.getAbsolutePath() + ". Parameter: " + name);
+					}
+					context.cache("previous-sql:" + db, content.get());
+					context.cache("db-version:" + db, name.substring("previous-sql:".length() + db.length()));
+					return Either.success(true);
+				}
+			}
+		}
+		return Either.success(false);
+	}
+
 	@Override
 	public boolean check(final Context context) {
 		if (context.contains(INSTANCE)) {
 			if (!context.contains(PostgresConnection.INSTANCE)
-					&& !context.contains(OracleConnection.INSTANCE)) {
+					&& !context.contains(OracleConnection.INSTANCE)
+					&& context.load("previous-sql:postgres") == null
+					&& context.load("previous-sql:oracle") == null) {
 				context.error("Connection string is required to create a migration script.\n"
 						+ "Neither Oracle or Postgres connection string found");
 				return false;
@@ -85,11 +127,11 @@ public void run(final Context context) throws ExitException {
 				context.error("Error accessing SQL path (" + path.getAbsolutePath() + ").");
 				throw new ExitException();
 			}
-			if (context.contains(PostgresConnection.INSTANCE)) {
+			if (context.load("previous-sql:postgres") instanceof String || context.contains(PostgresConnection.INSTANCE)) {
 				final DatabaseInfo dbInfo = PostgresConnection.getDatabaseDslAndVersion(context);
 				createMigration(context, path, dbInfo, POSTGRES_MIGRATION_FILE_NAME);
 			}
-			if (context.contains(OracleConnection.INSTANCE)) {
+			if (context.load("previous-sql:oracle") instanceof String || context.contains(OracleConnection.INSTANCE)) {
 				final DatabaseInfo dbInfo = OracleConnection.getDatabaseDslAndVersion(context);
 				createMigration(context, path, dbInfo, ORACLE_MIGRATION_FILE_NAME);
 			}
@@ -103,24 +145,49 @@ private static void createMigration(
 			final String file) throws ExitException {
 		final List<File> currentDsl = DslPath.getDslPaths(context);
 		context.show("Creating SQL migration for " + dbInfo.database + " ...");
+		final long start = new Date().getTime();
 		final Either<String> migration = DslCompiler.migration(context, dbInfo, currentDsl);
 		if (!migration.isSuccess()) {
 			context.error("Error creating SQL migration:");
 			context.error(migration.whyNot());
 			throw new ExitException();
 		}
+		final long end = new Date().getTime();
+		context.show("Running the migration took " + (end - start) / 1000 + " second(s)");
 		final String script = migration.get();
-		final String sqlFileName = dbInfo.database.toLowerCase() + "-sql-migration-" + (new Date().getTime());
+		final String customFile = context.get("sql:" + dbInfo.database.toLowerCase());
+		final String sqlFileName = customFile != null ? customFile : dbInfo.database.toLowerCase() + "-sql-migration-" + end + ".sql";
+		final File sqlFile = new File(path.getAbsolutePath(), sqlFileName);
+		boolean isContentSame = false;
+		if (customFile != null && sqlFile.exists()) {
+			Either<String> content = Utils.readFile(sqlFile);
+			isContentSame = content.isSuccess() && content.get().equals(script);
+			if (!content.isSuccess() || !isContentSame) {
+				if (context.contains(Force.INSTANCE)) {
+					context.show("Existing sql file (" + sqlFile.getAbsolutePath() + ") will be overwritten due to force option.");
+				} else if (!context.canInteract()) {
+					context.error("Custom sql migration file detected at: " + sqlFile.getAbsolutePath() + ". Enable force option, provide a different file name or delete the file for automatic migration");
+					throw new ExitException();
+				}
+				final String answer = context.ask("Existing sql migration file detected at: " + sqlFile.getAbsolutePath() + ". Do you wish to overwrite (y/N):");
+				if (!"y".equalsIgnoreCase(answer)) {
+					throw new ExitException();
+				}
+			}
+		}
 		if (script.length() > 0) {
-			final File sqlFile = new File(path.getAbsolutePath(), sqlFileName + ".sql");
-			try {
-				Utils.saveFile(context, sqlFile, script);
-			} catch (IOException e) {
-				context.error("Error saving migration script to " + sqlFile.getAbsolutePath());
-				context.error(e);
-				throw new ExitException();
+			if (!isContentSame) {
+				try {
+					Utils.saveFile(context, sqlFile, script);
+				} catch (IOException e) {
+					context.error("Error saving migration script to " + sqlFile.getAbsolutePath());
+					context.error(e);
+					throw new ExitException();
+				}
+				context.show("Migration saved to " + sqlFile.getAbsolutePath());
+			} else {
+				context.show("Sql migration remains same as before in: " + sqlFile.getAbsolutePath());
 			}
-			context.show("Migration saved to " + sqlFile.getAbsolutePath());
 			final String[] descriptions = extractDescriptions(script);
 			for (int i = 1; i < descriptions.length; i++) {
 				context.log(descriptions[i]);
@@ -142,10 +209,10 @@ public String getDetailedDescription() {
 		return "DSL Platform will compare previously applied DSL with the current one and provide a migration SQL script.\n" +
 				"Developer can inspect migration (although it contains a lot of boilerplate due to dependency graph rebuild),\n" +
 				"to check if the requested migration matches what he had in mind.\n" +
-				"Every migration contains description of the important changes to the database.\n" +
-				"\n" +
-				"Postgres migrations are transactional due to Transactional DDL Postgres feature.\n" +
-				"\n" +
-				"While for most migrations ownership of the database is sufficient, some require superuser access (Enum changes, strange primary keys, ...).";
+				"Every migration contains description of the important changes to the database.\n\n" +
+				"Postgres migrations are transactional due to Transactional DDL Postgres feature.\n\n" +
+				"While for most migrations ownership of the database is sufficient, some require superuser access (Enum changes, strange primary keys, ...).\n\n" +
+				"Custom sql files can be specified via sql:[database] file, eg. sql:postgres=03-dsl-migration.sql.\n\n" +
+				"To avoid using database, previous sql file can be specified via previous-sql:[databaseVersion] file, eg. previous-sql:postgres9.6=02-dsl-migration.sql.\n\n";
 	}
 }

CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/OracleConnection.java

@@ -55,6 +55,10 @@ public static DatabaseInfo getDatabaseDslAndVersion(final Context context) throw
 		if (cache != null) {
 			return cache;
 		}
+		final String previous = context.load("previous-sql:oracle");
+		if (previous != null) {
+			return extractDatabaseInfoFromMigration(context, previous);
+		}
 		final String value = context.get(INSTANCE);
 		final String connectionString = "jdbc:oracle:thin:" + value;
 		Connection conn;
@@ -133,6 +137,56 @@ public static DatabaseInfo getDatabaseDslAndVersion(final Context context) throw
 		return emptyResult;
 	}
 
+	static DatabaseInfo extractDatabaseInfoFromMigration(final Context context, final String previous) throws ExitException {
+		final String dbVersion = context.load("db-version:oracle");
+		final String pattern = "INSERT INTO \"-DSL-\".Database_Migration (Ordinal, Dsls, Version) VALUES(\"-DSL-\".DM_SEQ.nextval,";
+		final int persistInd = previous.lastIndexOf(pattern);
+		if (persistInd == -1) {
+			context.error("Unable to find INSERT INTO \"-DSL-\".Database_Migration in previous sql migration. Wrong file provided");
+			throw new ExitException();
+		}
+		final String lastDsl;
+		final int patternSimpleStart = previous.indexOf(pattern + " '");
+		final int patternComplexStart = previous.indexOf(pattern + " dsls, '");
+		final String afterPersist = previous.substring(persistInd);
+		final int compilerEnd = afterPersist.lastIndexOf('\'');
+		final int compilerStart = compilerEnd == -1 ? -1 : afterPersist.substring(0, compilerEnd - 1).lastIndexOf('\'');
+		final String compiler = compilerStart == -1 ? "" : afterPersist.substring(compilerStart + 1, compilerEnd);
+		if (compiler.length() == 0) {
+			context.error("Unable to find appropriate compiler info after INSERT INTO \"-DSL-\".Database_Migration in previous sql migration. Wrong file provided");
+			throw new ExitException();
+		} else if (patternSimpleStart >= 0) {
+			lastDsl = afterPersist.substring(pattern.length() + 2, compilerStart - 3);
+		} else if (patternComplexStart >= 0) {
+			final StringBuilder dsls = new StringBuilder();
+			int clobStart = previous.indexOf("dsls := '");
+			if (clobStart == -1) {
+				context.error("Unable to find appropriate dsls := in previous sql migration. Wrong file provided");
+				throw new ExitException();
+			}
+			clobStart += "dsls := '".length();
+			int clobEnd = previous.indexOf("dsls := dsls || '");
+			while (clobEnd != -1) {
+				final String part = previous.substring(clobStart, clobEnd);
+				final int scEnd = part.lastIndexOf('\'');
+				dsls.append(part, 0, scEnd);
+				clobStart = clobEnd + "dsls := dsls || '".length();
+				clobEnd = previous.indexOf("dsls := dsls || '", clobStart);
+			}
+			final String lastPart = previous.substring(clobStart, persistInd);
+			final int scEnd = lastPart.lastIndexOf('\'');
+			dsls.append(lastPart, 0, scEnd);
+			lastDsl = dsls.toString();
+		} else {
+			context.error("Unable to find appropriate INSERT INTO \"-DSL-\".Database_Migration in previous sql migration. Wrong file provided");
+			throw new ExitException();
+		}
+		final Map<String, String> dslMap = DatabaseInfo.convertToMap(lastDsl.replace("''", "'"), context);
+		final DatabaseInfo result = new DatabaseInfo("Oracle", compiler, dbVersion, dslMap);
+		context.cache(CACHE_NAME, result);
+		return result;
+	}
+
 	public static void execute(final Context context, final String sql) throws ExitException {
 		final String value = context.get(INSTANCE);
 		final String connectionString = "jdbc:oracle:thin:" + value;

CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/PostgresConnection.java

@@ -41,6 +41,10 @@ public static DatabaseInfo getDatabaseDslAndVersion(final Context context) throw
 		if (cache != null) {
 			return cache;
 		}
+		final String previous = context.load("previous-sql:postgres");
+		if (previous != null) {
+			return extractDatabaseInfoFromMigration(context, previous);
+		}
 		final String value = context.get(INSTANCE);
 		final String connectionString = "jdbc:postgresql://" + value;
 		Connection conn;
@@ -120,6 +124,29 @@ public static DatabaseInfo getDatabaseDslAndVersion(final Context context) throw
 		return emptyResult;
 	}
 
+	static DatabaseInfo extractDatabaseInfoFromMigration(final Context context, final String previous) throws ExitException {
+		final String dbVersion = context.load("db-version:postgres");
+		final int persistInd = previous.lastIndexOf("SELECT \"-DSL-\".Persist_Concepts('");
+		final int notifyInd = previous.indexOf("SELECT pg_notify", persistInd + 1);
+		if (persistInd == -1 || notifyInd == -1) {
+			context.error("Unable to find 'Persist_Concepts' or SELECT pg_notify in previous sql migration. Wrong file provided");
+			throw new ExitException();
+		}
+		final String subset = previous.substring(persistInd + "SELECT \"-DSL-\".Persist_Concepts(".length() + 1, notifyInd - 2);
+		final String pattern = "\"', '\\x','";
+		final int lastNL = subset.lastIndexOf(pattern);
+		if (lastNL == -1) {
+			context.error("Invalid content detected in previous sql migration. Unable to find magic pattern: " + pattern);
+			throw new ExitException();
+		}
+		final String compiler = subset.substring(lastNL + pattern.length(), subset.lastIndexOf('\''));
+		final String lastDsl = subset.substring(0, lastNL + 1).replace("''", "'");
+		final Map<String, String> dslMap = DatabaseInfo.convertToMap(lastDsl, context);
+		final DatabaseInfo result = new DatabaseInfo("Postgres", compiler, dbVersion, dslMap);
+		context.cache(CACHE_NAME, result);
+		return result;
+	}
+
 	public static void execute(final Context context, final String sql) throws ExitException {
 		final String connectionString = "jdbc:postgresql://" + context.get(INSTANCE);
 

CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/SqlPath.java

@@ -27,7 +27,7 @@ public boolean check(final Context context) throws ExitException {
 				final File sqlPath = new File(value);
 				if (!sqlPath.exists()) {
 					if (!context.contains(Force.INSTANCE)) {
-						context.warning("SQL path provided (" + sqlPath.getAbsolutePath() + ") but doesn't exists.");
+						context.warning("SQL path provided (" + sqlPath.getAbsolutePath() + ") but doesn't exist.");
 						if (!context.canInteract()) {
 							context.error("Specify existing path or remove parameter to use temporary folder.");
 							return false;

CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/Targets.java

@@ -263,8 +263,15 @@ private void compile(final Context context, final List<Option> targets) throws E
 							dsls,
 							context.get("library:" + t.value));
 			try {
+				boolean hasFileWithExtension = false;
+				for (final String name : files.keySet()) {
+					if (name.endsWith(t.extension)) {
+						hasFileWithExtension = true;
+						break;
+					}
+				}
 				for (final Map.Entry<String, String> kv : files.entrySet()) {
-					final String fullName = t.name() + "/" + kv.getKey() + t.extension;
+					final String fullName = t.name() + "/" + kv.getKey() + (hasFileWithExtension ? "" : t.extension);
 					saveFile(context, temp, t.convertToPath, fullName, kv.getValue());
 				}
 			} catch (IOException e) {

CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/TempPath.java

@@ -90,7 +90,7 @@ public boolean check(final Context context) {
 				final File path = new File(value);
 				if (!path.exists()) {
 					if (!context.contains(Force.INSTANCE)) {
-						context.error("Temporary path provided (" + value + "), but doesn't exists. Please create it or use system path.");
+						context.error("Temporary path provided (" + value + "), but doesn't exist. Please create it or use system path.");
 						return false;
 					} else {
 						context.show("Due to force option enabled, creating temp folder in: " + path.getAbsolutePath());