diff --git a/CommandLineClient/pom.xml b/CommandLineClient/pom.xml index ec9b531..76b4f96 100644 --- a/CommandLineClient/pom.xml +++ b/CommandLineClient/pom.xml @@ -4,7 +4,7 @@ com.dslplatform dsl-clc jar - 1.9.4 + 1.9.5 DSL Platform - Compiler Command-Line Client https://github.com/ngs-doo/dsl-compiler-client Command line client for interaction with DSL Platform compiler (https://dsl-platform.com) diff --git a/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/ApplyMigration.java b/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/ApplyMigration.java index 0be5038..46fa8b6 100644 --- a/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/ApplyMigration.java +++ b/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/ApplyMigration.java @@ -133,7 +133,10 @@ private static void applyMigrationScript(final Context context, final File file, } context.show("Applying migration..."); db.execute(context, sql); - if (file.renameTo(new File(file.getParentFile(), "applied-" + file.getName()))) { + final String customFile = context.get("sql:" + db.getDName().toLowerCase()); + if (customFile != null && file.getName().equals(customFile)) { + context.show("Database migrated via: " + file.getAbsolutePath()); + } else if (file.renameTo(new File(file.getParentFile(), "applied-" + file.getName()))) { context.show("Database migrated and script renamed to: applied-" + file.getName()); } else { context.show("Database migrated, but unable to rename script: " + file.getName()); diff --git a/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/DslCompiler.java b/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/DslCompiler.java index b272a53..58962f3 100644 --- a/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/DslCompiler.java +++ b/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/DslCompiler.java @@ -56,11 +56,15 @@ public static Map compile( arguments.add("dsl=" + f.getAbsolutePath()); } context.log("Compiling DSL to " + target + "..."); + final long start = new Date().getTime(); final Either response = runCompiler(context, arguments); if (!response.isSuccess()) { context.error(response.whyNot()); throw new ExitException(); } + final long end = new Date().getTime(); + context.show("Creating the source took " + (end - start) / 1000 + " second(s)"); + final Either xml = Utils.readXml(new ByteArrayInputStream(response.get())); if (!xml.isSuccess()) { context.error(new String(response.get(), UTF_8)); @@ -635,7 +639,7 @@ public static File lookupDefaultPath(final Context context) throws ExitException final String dslName = Download.isDefaultUrl(context) ? ".DSL-Platform" : ".DSL-Custom"; final File compilerFolder = new File(homePath, dslName); if (!compilerFolder.exists() && !compilerFolder.mkdirs()) { - context.warning("Error creating dsl compiler path in: " + compilerFolder.getAbsolutePath() + ". Will use temporary filter instead."); + context.warning("Error creating dsl compiler path in: " + compilerFolder.getAbsolutePath() + ". Will use temporary folder instead."); return new File(TempPath.getTempRootPath(context), "dsl-compiler.exe"); } return new File(compilerFolder, "dsl-compiler.exe"); diff --git a/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/DslPath.java b/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/DslPath.java index c50001c..affe87b 100644 --- a/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/DslPath.java +++ b/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/DslPath.java @@ -43,7 +43,7 @@ private static void findDsls(final Context context) throws ExitException { String value = context.get(INSTANCE); if (value == null) { if (!(new File("./dsl").exists())) { - context.error("DSL path not provided. Can't use default path (./dsl) since it doesn't exists"); + context.error("DSL path not provided. Can't use default path (./dsl) since it doesn't exist"); throw new ExitException(); } context.put(INSTANCE, value = "./dsl"); @@ -88,7 +88,7 @@ public boolean check(final Context context) { if (value == null) { final File dslPath = new File("./dsl"); if (!dslPath.exists()) { - context.error("DSL path not provided. Can't use default path (./dsl) since it doesn't exists"); + context.error("DSL path not provided. Can't use default path (./dsl) since it doesn't exist"); return false; } context.put(INSTANCE, "./dsl"); @@ -96,7 +96,7 @@ public boolean check(final Context context) { for (final String part : value.split(File.pathSeparator)) { final File dslPath = new File(part).getAbsoluteFile(); if (!dslPath.exists()) { - context.error("Provided DSL path (" + part + ") does not exists. Please provide valid path to DSL files"); + context.error("Provided DSL path (" + part + ") does not exist. Please provide valid path to DSL files"); return false; } } diff --git a/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/Migration.java b/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/Migration.java index e309c88..f414d6f 100644 --- a/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/Migration.java +++ b/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/Migration.java @@ -6,7 +6,7 @@ import java.io.IOException; import java.util.*; -public enum Migration implements CompileParameter { +public enum Migration implements CompileParameter, ParameterParser { INSTANCE; @Override @@ -33,7 +33,7 @@ public static File getOracleMigrationFile(final Context context) { return context.load(ORACLE_MIGRATION_FILE_NAME); } - public static String[] extractDescriptions(final String sql) throws ExitException { + public static String[] extractDescriptions(final String sql) { final int start = sql.indexOf(DESCRIPTION_START); final int end = sql.indexOf(DESCRIPTION_END); if (end > start) { @@ -42,11 +42,53 @@ public static String[] extractDescriptions(final String sql) throws ExitExceptio return new String[0]; } + @Override + public Either tryParse(final String name, final String value, final Context context) { + if ("migration".equals(name)) { + context.put(name, value == null || value.length() == 0 ? null : value); + return Either.success(true); + } else { + for (final String db : new String[]{"postgres", "oracle"}) { + if (("sql:" + db).equalsIgnoreCase(name)) { + if (value == null || value.length() == 0) { + return Either.fail("Custom output file parameter detected, but it's missing file name as an argument. Parameter: " + name); + } + final File path = new File(value); + if (path.exists() && path.isDirectory()) { + return Either.fail("Output path found, but it's a directory. Parameter: " + name); + } + context.put("sql:" + db, value); + return Either.success(true); + } else if (name.startsWith("previous-sql:" + db)) { + if (value == null || value.length() == 0) { + return Either.fail("Previous sql file parameter detected, but it's missing path as an argument. Parameter: " + name); + } + final File previous = new File(value); + if (!previous.exists()) { + return Either.fail("Previous sql path provided, but file does not exist at: " + previous.getAbsolutePath() + ". Parameter: " + name); + } else if (previous.isDirectory()) { + return Either.fail("Previous sql path found, but it's a directory: " + previous.getAbsolutePath() + ". Parameter: " + name); + } + final Either content = Utils.readFile(previous); + if (!content.isSuccess()) { + return Either.fail("Unable to read previous sql file from: " + previous.getAbsolutePath() + ". Parameter: " + name); + } + context.cache("previous-sql:" + db, content.get()); + context.cache("db-version:" + db, name.substring("previous-sql:".length() + db.length())); + return Either.success(true); + } + } + } + return Either.success(false); + } + @Override public boolean check(final Context context) { if (context.contains(INSTANCE)) { if (!context.contains(PostgresConnection.INSTANCE) - && !context.contains(OracleConnection.INSTANCE)) { + && !context.contains(OracleConnection.INSTANCE) + && context.load("previous-sql:postgres") == null + && context.load("previous-sql:oracle") == null) { context.error("Connection string is required to create a migration script.\n" + "Neither Oracle or Postgres connection string found"); return false; @@ -85,11 +127,11 @@ public void run(final Context context) throws ExitException { context.error("Error accessing SQL path (" + path.getAbsolutePath() + ")."); throw new ExitException(); } - if (context.contains(PostgresConnection.INSTANCE)) { + if (context.load("previous-sql:postgres") instanceof String || context.contains(PostgresConnection.INSTANCE)) { final DatabaseInfo dbInfo = PostgresConnection.getDatabaseDslAndVersion(context); createMigration(context, path, dbInfo, POSTGRES_MIGRATION_FILE_NAME); } - if (context.contains(OracleConnection.INSTANCE)) { + if (context.load("previous-sql:oracle") instanceof String || context.contains(OracleConnection.INSTANCE)) { final DatabaseInfo dbInfo = OracleConnection.getDatabaseDslAndVersion(context); createMigration(context, path, dbInfo, ORACLE_MIGRATION_FILE_NAME); } @@ -103,24 +145,49 @@ private static void createMigration( final String file) throws ExitException { final List currentDsl = DslPath.getDslPaths(context); context.show("Creating SQL migration for " + dbInfo.database + " ..."); + final long start = new Date().getTime(); final Either migration = DslCompiler.migration(context, dbInfo, currentDsl); if (!migration.isSuccess()) { context.error("Error creating SQL migration:"); context.error(migration.whyNot()); throw new ExitException(); } + final long end = new Date().getTime(); + context.show("Running the migration took " + (end - start) / 1000 + " second(s)"); final String script = migration.get(); - final String sqlFileName = dbInfo.database.toLowerCase() + "-sql-migration-" + (new Date().getTime()); + final String customFile = context.get("sql:" + dbInfo.database.toLowerCase()); + final String sqlFileName = customFile != null ? customFile : dbInfo.database.toLowerCase() + "-sql-migration-" + end + ".sql"; + final File sqlFile = new File(path.getAbsolutePath(), sqlFileName); + boolean isContentSame = false; + if (customFile != null && sqlFile.exists()) { + Either content = Utils.readFile(sqlFile); + isContentSame = content.isSuccess() && content.get().equals(script); + if (!content.isSuccess() || !isContentSame) { + if (context.contains(Force.INSTANCE)) { + context.show("Existing sql file (" + sqlFile.getAbsolutePath() + ") will be overwritten due to force option."); + } else if (!context.canInteract()) { + context.error("Custom sql migration file detected at: " + sqlFile.getAbsolutePath() + ". Enable force option, provide a different file name or delete the file for automatic migration"); + throw new ExitException(); + } + final String answer = context.ask("Existing sql migration file detected at: " + sqlFile.getAbsolutePath() + ". Do you wish to overwrite (y/N):"); + if (!"y".equalsIgnoreCase(answer)) { + throw new ExitException(); + } + } + } if (script.length() > 0) { - final File sqlFile = new File(path.getAbsolutePath(), sqlFileName + ".sql"); - try { - Utils.saveFile(context, sqlFile, script); - } catch (IOException e) { - context.error("Error saving migration script to " + sqlFile.getAbsolutePath()); - context.error(e); - throw new ExitException(); + if (!isContentSame) { + try { + Utils.saveFile(context, sqlFile, script); + } catch (IOException e) { + context.error("Error saving migration script to " + sqlFile.getAbsolutePath()); + context.error(e); + throw new ExitException(); + } + context.show("Migration saved to " + sqlFile.getAbsolutePath()); + } else { + context.show("Sql migration remains same as before in: " + sqlFile.getAbsolutePath()); } - context.show("Migration saved to " + sqlFile.getAbsolutePath()); final String[] descriptions = extractDescriptions(script); for (int i = 1; i < descriptions.length; i++) { context.log(descriptions[i]); @@ -142,10 +209,10 @@ public String getDetailedDescription() { return "DSL Platform will compare previously applied DSL with the current one and provide a migration SQL script.\n" + "Developer can inspect migration (although it contains a lot of boilerplate due to dependency graph rebuild),\n" + "to check if the requested migration matches what he had in mind.\n" + - "Every migration contains description of the important changes to the database.\n" + - "\n" + - "Postgres migrations are transactional due to Transactional DDL Postgres feature.\n" + - "\n" + - "While for most migrations ownership of the database is sufficient, some require superuser access (Enum changes, strange primary keys, ...)."; + "Every migration contains description of the important changes to the database.\n\n" + + "Postgres migrations are transactional due to Transactional DDL Postgres feature.\n\n" + + "While for most migrations ownership of the database is sufficient, some require superuser access (Enum changes, strange primary keys, ...).\n\n" + + "Custom sql files can be specified via sql:[database] file, eg. sql:postgres=03-dsl-migration.sql.\n\n" + + "To avoid using database, previous sql file can be specified via previous-sql:[databaseVersion] file, eg. previous-sql:postgres9.6=02-dsl-migration.sql.\n\n"; } } diff --git a/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/OracleConnection.java b/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/OracleConnection.java index 1d5c459..5be9842 100644 --- a/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/OracleConnection.java +++ b/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/OracleConnection.java @@ -55,6 +55,10 @@ public static DatabaseInfo getDatabaseDslAndVersion(final Context context) throw if (cache != null) { return cache; } + final String previous = context.load("previous-sql:oracle"); + if (previous != null) { + return extractDatabaseInfoFromMigration(context, previous); + } final String value = context.get(INSTANCE); final String connectionString = "jdbc:oracle:thin:" + value; Connection conn; @@ -133,6 +137,56 @@ public static DatabaseInfo getDatabaseDslAndVersion(final Context context) throw return emptyResult; } + static DatabaseInfo extractDatabaseInfoFromMigration(final Context context, final String previous) throws ExitException { + final String dbVersion = context.load("db-version:oracle"); + final String pattern = "INSERT INTO \"-DSL-\".Database_Migration (Ordinal, Dsls, Version) VALUES(\"-DSL-\".DM_SEQ.nextval,"; + final int persistInd = previous.lastIndexOf(pattern); + if (persistInd == -1) { + context.error("Unable to find INSERT INTO \"-DSL-\".Database_Migration in previous sql migration. Wrong file provided"); + throw new ExitException(); + } + final String lastDsl; + final int patternSimpleStart = previous.indexOf(pattern + " '"); + final int patternComplexStart = previous.indexOf(pattern + " dsls, '"); + final String afterPersist = previous.substring(persistInd); + final int compilerEnd = afterPersist.lastIndexOf('\''); + final int compilerStart = compilerEnd == -1 ? -1 : afterPersist.substring(0, compilerEnd - 1).lastIndexOf('\''); + final String compiler = compilerStart == -1 ? "" : afterPersist.substring(compilerStart + 1, compilerEnd); + if (compiler.length() == 0) { + context.error("Unable to find appropriate compiler info after INSERT INTO \"-DSL-\".Database_Migration in previous sql migration. Wrong file provided"); + throw new ExitException(); + } else if (patternSimpleStart >= 0) { + lastDsl = afterPersist.substring(pattern.length() + 2, compilerStart - 3); + } else if (patternComplexStart >= 0) { + final StringBuilder dsls = new StringBuilder(); + int clobStart = previous.indexOf("dsls := '"); + if (clobStart == -1) { + context.error("Unable to find appropriate dsls := in previous sql migration. Wrong file provided"); + throw new ExitException(); + } + clobStart += "dsls := '".length(); + int clobEnd = previous.indexOf("dsls := dsls || '"); + while (clobEnd != -1) { + final String part = previous.substring(clobStart, clobEnd); + final int scEnd = part.lastIndexOf('\''); + dsls.append(part, 0, scEnd); + clobStart = clobEnd + "dsls := dsls || '".length(); + clobEnd = previous.indexOf("dsls := dsls || '", clobStart); + } + final String lastPart = previous.substring(clobStart, persistInd); + final int scEnd = lastPart.lastIndexOf('\''); + dsls.append(lastPart, 0, scEnd); + lastDsl = dsls.toString(); + } else { + context.error("Unable to find appropriate INSERT INTO \"-DSL-\".Database_Migration in previous sql migration. Wrong file provided"); + throw new ExitException(); + } + final Map dslMap = DatabaseInfo.convertToMap(lastDsl.replace("''", "'"), context); + final DatabaseInfo result = new DatabaseInfo("Oracle", compiler, dbVersion, dslMap); + context.cache(CACHE_NAME, result); + return result; + } + public static void execute(final Context context, final String sql) throws ExitException { final String value = context.get(INSTANCE); final String connectionString = "jdbc:oracle:thin:" + value; diff --git a/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/PostgresConnection.java b/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/PostgresConnection.java index f541189..e6f8cbc 100644 --- a/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/PostgresConnection.java +++ b/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/PostgresConnection.java @@ -41,6 +41,10 @@ public static DatabaseInfo getDatabaseDslAndVersion(final Context context) throw if (cache != null) { return cache; } + final String previous = context.load("previous-sql:postgres"); + if (previous != null) { + return extractDatabaseInfoFromMigration(context, previous); + } final String value = context.get(INSTANCE); final String connectionString = "jdbc:postgresql://" + value; Connection conn; @@ -120,6 +124,29 @@ public static DatabaseInfo getDatabaseDslAndVersion(final Context context) throw return emptyResult; } + static DatabaseInfo extractDatabaseInfoFromMigration(final Context context, final String previous) throws ExitException { + final String dbVersion = context.load("db-version:postgres"); + final int persistInd = previous.lastIndexOf("SELECT \"-DSL-\".Persist_Concepts('"); + final int notifyInd = previous.indexOf("SELECT pg_notify", persistInd + 1); + if (persistInd == -1 || notifyInd == -1) { + context.error("Unable to find 'Persist_Concepts' or SELECT pg_notify in previous sql migration. Wrong file provided"); + throw new ExitException(); + } + final String subset = previous.substring(persistInd + "SELECT \"-DSL-\".Persist_Concepts(".length() + 1, notifyInd - 2); + final String pattern = "\"', '\\x','"; + final int lastNL = subset.lastIndexOf(pattern); + if (lastNL == -1) { + context.error("Invalid content detected in previous sql migration. Unable to find magic pattern: " + pattern); + throw new ExitException(); + } + final String compiler = subset.substring(lastNL + pattern.length(), subset.lastIndexOf('\'')); + final String lastDsl = subset.substring(0, lastNL + 1).replace("''", "'"); + final Map dslMap = DatabaseInfo.convertToMap(lastDsl, context); + final DatabaseInfo result = new DatabaseInfo("Postgres", compiler, dbVersion, dslMap); + context.cache(CACHE_NAME, result); + return result; + } + public static void execute(final Context context, final String sql) throws ExitException { final String connectionString = "jdbc:postgresql://" + context.get(INSTANCE); diff --git a/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/SqlPath.java b/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/SqlPath.java index 21c8e5b..952c604 100644 --- a/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/SqlPath.java +++ b/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/SqlPath.java @@ -27,7 +27,7 @@ public boolean check(final Context context) throws ExitException { final File sqlPath = new File(value); if (!sqlPath.exists()) { if (!context.contains(Force.INSTANCE)) { - context.warning("SQL path provided (" + sqlPath.getAbsolutePath() + ") but doesn't exists."); + context.warning("SQL path provided (" + sqlPath.getAbsolutePath() + ") but doesn't exist."); if (!context.canInteract()) { context.error("Specify existing path or remove parameter to use temporary folder."); return false; diff --git a/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/Targets.java b/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/Targets.java index caaf40c..324a53f 100644 --- a/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/Targets.java +++ b/CommandLineClient/src/main/java/com/dslplatform/compiler/client/parameters/Targets.java @@ -263,8 +263,15 @@ private void compile(final Context context, final List