Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve documentation and performance of CSync #283

Open
wants to merge 7 commits into
base: develop
Choose a base branch
from
+106 −65
Open

Improve documentation and performance of CSync #283

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d8f7cae
add enriched svg capability back to MDK
Enquier May 28, 2024
60ca27d
replace twcId inside svg
Enquier May 28, 2024
90a2420
minor tweaks for 2021x compatibility
Enquier May 28, 2024
9f8b5cd
continue to improve runner documentation and performance
Enquier May 28, 2024
136eccd
fix javafx
Enquier May 28, 2024
83dc174
fix branch validation/twcIds
Enquier May 29, 2024
eb74079
Merge commit '50f44ca842999b7ef106b0c33b00755a926ea6d7' into bugfix/p…
Jun 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,10 @@ dependencies {

implementation group: 'com.beust', name: 'jcommander', version: '1.72'

// jsoup HTML parser library @ https://jsoup.org/
implementation group: 'org.jsoup', name: 'jsoup', version: '1.17.2'


// Test Dependencies
//testImplementation 'junit:junit:4.11'
testImplementation group: 'org.gradle', name: 'gradle-core', version: '6.1.1'
Expand Down Expand Up @@ -225,10 +229,13 @@ dependencies {
distribution group: 'org.openjfx', name: 'javafx-fxml', version: '11.0.2', classifier: 'win'
distribution group: 'org.openjfx', name: 'javafx-controls', version: '11.0.2', classifier: 'win'
distribution group: 'org.openjfx', name: 'javafx-swing', version: '11.0.2', classifier: 'win'
distribution group: 'org.openjfx', name: 'javafx-graphics', version: '11.0.2', classifier: 'win'
distribution group: 'org.openjfx', name: 'javafx-base', version: '11.0.2', classifier: 'win'
distribution group: 'org.openjfx', name: 'javafx-fxml', version: '11.0.2', classifier: 'mac'
distribution group: 'org.openjfx', name: 'javafx-controls', version: '11.0.2', classifier: 'mac'
distribution group: 'org.openjfx', name: 'javafx-swing', version: '11.0.2', classifier: 'mac'

distribution group: 'org.openjfx', name: 'javafx-graphics', version: '11.0.2', classifier: 'mac'
distribution group: 'org.openjfx', name: 'javafx-base', version: '11.0.2', classifier: 'mac'

// This ensures classpath load order to match the MagicDraw provided order and then includes extras needed for non-OpenAPI stuff.
// This was necessary because of the Application class stubbing that was done in the chromium libraries.
Expand Down Expand Up @@ -292,15 +299,15 @@ task extractDependencies {
from 'build/dependency-cache/extracted/magicdraw/plugins/com.nomagic.conceptmodeler'
into 'build/tmp/com.nomagic.conceptmodeler'
}

}
}
fileTree(dir: 'build/dependency-cache/extracted/magicdraw', include: '*.zip').each { archive ->
copy {
from zipTree(archive)
into 'build/dependency-cache/extracted/magicdraw'
}

}
delete 'build/dependency-cache/extracted/magicdraw/plugins/com.nomagic.conceptmodeler'
}
Expand Down Expand Up @@ -579,7 +586,7 @@ nexusPublishing {

artifactoryPublish {
publications('mavenJava')

}

artifactoryPublish.dependsOn distZip
Expand Down
2 changes: 1 addition & 1 deletion gradle.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
version=6.1.0
version=6.1.1
group=org.openmbee.mdk.magic
descriptorFile=MDR_Plugin_Model_Development_Kit_91110_descriptor.xml
magicdDrawGroupName=org.openmbee.mdk
Expand Down
1 change: 1 addition & 0 deletions src/main/java/org/openmbee/mdk/api/incubating/MDKConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ public class MDKConstants {
MESSAGES_NODE = "messages",
REJECTED_NODE = "rejected",
SOURCE_FIELD = "source",
ARCHIVED_FIELD = "_archived",
DELETED_FIELD = "deleted",
MAGICDRAW_SOURCE_VALUE = "magicdraw",
ELEMENT_TYPE_VALUE = "Element",
Expand Down
13 changes: 12 additions & 1 deletion src/main/java/org/openmbee/mdk/emf/EMFExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,15 @@
import org.openmbee.mdk.json.JacksonUtils;
import org.openmbee.mdk.util.MDUtils;
import org.openmbee.mdk.util.Utils;
import org.w3c.dom.html.HTMLDocument;
import org.eclipse.emf.ecore.EDataType;
import org.eclipse.emf.ecore.EObject;
import org.eclipse.emf.ecore.EReference;
import org.eclipse.emf.ecore.EStructuralFeature;
import org.eclipse.emf.ecore.util.EcoreUtil;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
Expand All @@ -39,6 +43,7 @@
import java.util.function.BiFunction;
import java.util.stream.Collectors;
import java.io.Reader;
import java.io.StringReader;

public class EMFExporter implements BiFunction<Element, Project, ObjectNode> {
@Override
Expand Down Expand Up @@ -161,7 +166,13 @@ private enum Processor {
DOCUMENTATION_PRE(
(element, project, objectNode) -> {
//Todo: FIgure out how to read this stuff and parse the <a tags from MD
// String doc_text = (String) Utils.getElementAttribute(element, Utils.AvailableAttribute.Documentation);
String doc_text = (String) Utils.getElementAttribute(element, Utils.AvailableAttribute.Documentation);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

don't think this is completely implemented? the parsed doc isn't used anywhere, there's no mms-cf attr added, and should have a try catch surrounding it in case jsoup fails, also it'll lead to a diff since the output on mms will then be different to the model, think this should commented out and revisited later

Document doc = Jsoup.parse(doc_text);
Elements crossReferences = doc.select("a[href^=mdel:]");
if (!crossReferences.isEmpty()) {
crossReferences.tagName("mms-cf");

}
// Reader reader = new StringReader(doc_text);
// HTMLDocument html = new HTMLDocument(doc_text);
objectNode.put(MDKConstants.DOCUMENTATION_KEY, (String) Utils.getElementAttribute(element, Utils.AvailableAttribute.Documentation));
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/org/openmbee/mdk/mms/actions/ValidateBranchesAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ public class ValidationRunner implements RunnableWithProgress {

@Override
public void run(ProgressStatus arg0) {
BranchValidator branchValidator = new BranchValidator(Application.getInstance().getProject());
branchValidator.validate(arg0, true);
BranchValidator branchValidator = new BranchValidator(Application.getInstance().getProject(), true);
branchValidator.run(arg0);
if (branchValidator.hasErrors()) {
Application.getInstance().getGUILog().log("[ERROR] Unable to complete validate branches action.");
return;
Expand Down
70 changes: 43 additions & 27 deletions src/main/java/org/openmbee/mdk/mms/sync/delta/DeltaSyncRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import com.nomagic.magicdraw.openapi.uml.SessionManager;
import com.nomagic.task.ProgressStatus;
import com.nomagic.task.RunnableWithProgress;
import com.nomagic.ui.ProgressStatusRunner;
import com.nomagic.uml2.ext.magicdraw.classes.mdkernel.Element;
import com.nomagic.uml2.ext.magicdraw.classes.mdkernel.ValueSpecification;
import org.openmbee.mdk.api.incubating.MDKConstants;
Expand Down Expand Up @@ -59,12 +60,16 @@ public DeltaSyncRunner(boolean shouldCommit, boolean shouldCommitDeletes, boolea
this.shouldCommit = shouldCommit;
this.shouldCommitDeletes = shouldCommitDeletes;
this.shouldUpdate = shouldUpdate;
}

}
@SuppressWarnings("unchecked")
@Override
public void run(ProgressStatus progressStatus) {
progressStatus.setDescription("Initializing");
progressStatus.setDescription("Coordinated Sync: Initializing");
progressStatus.setCurrent(0);
progressStatus.setIndeterminate(false);
progressStatus.setMax(15);
progressStatus.cancelIfCanceled();
if (ProjectUtilities.isFromEsiServer(project.getPrimaryProject()) && EsiUtils.getLoggedUserName() == null) {
Utils.guilog("[WARNING] You need to be logged in to Teamwork Cloud first. Skipping sync. All changes will be persisted in the model and re-attempted in the next sync.");
return;
Expand All @@ -78,9 +83,13 @@ public void run(ProgressStatus progressStatus) {
Utils.guilog("[ERROR] An error occurred while validating credentials. Credentials will be cleared. Skipping sync. All changes will be persisted in the model and re-attempted in the next sync. Reason: " + e.getMessage());
return;
}
progressStatus.increase();
progressStatus.setDescription("Coordinated Sync: Project Validation");
progressStatus.cancelIfCanceled();

ProjectValidator pv = new ProjectValidator(project);
pv.validate();
ProgressStatusRunner.runWithProgressStatus(pv, "Coordinated Sync: Project Validation", true, 0);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is the reason why it's starting a new runner instead of calling pv.run(progressStatus) because it's changing to indeterminate?


if (pv.hasErrors()) {
Application.getInstance().getGUILog().log("[WARNING] Coordinated Sync can not complete and will be skipped.");
return;
Expand All @@ -90,9 +99,12 @@ public void run(ProgressStatus progressStatus) {
Utils.displayValidationWindow(project, pv.getValidationSuite(), "Coordinated Sync Pre-Condition Validation");
return;
}

BranchValidator bv = new BranchValidator(project);
bv.validate(null, false);
progressStatus.increase();
progressStatus.setDescription("Coordinated Sync: Branch Validation");
progressStatus.cancelIfCanceled();

BranchValidator bv = new BranchValidator(project, false);
ProgressStatusRunner.runWithProgressStatus(bv, "Coordinated Sync: Branch Validation", true, 0);
if (bv.hasErrors()) {
Application.getInstance().getGUILog().log("[WARNING] Coordinated sync can not complete and will be skipped.");
return;
Expand All @@ -102,15 +114,16 @@ public void run(ProgressStatus progressStatus) {
Utils.displayValidationWindow(project, bv.getValidationSuite(), "Coordinated Sync Pre-Condition Validation");
return;
}

progressStatus.increase();
LocalDeltaTransactionCommitListener listener = LocalDeltaProjectEventListenerAdapter.getProjectMapping(project).getLocalDeltaTransactionCommitListener();

// UPDATE LOCKS

listener.setDisabled(true);

// UPDATE MMS CHANGELOG

progressStatus.setDescription("Coordinated Sync: Updating Server Changelog");
progressStatus.cancelIfCanceled();
try {
if (!MMSDeltaProjectEventListenerAdapter.getProjectMapping(project).update()) {
Application.getInstance().getGUILog().log("[WARNING] MMS history is unavailable. Skipping sync. All changes will be re-attempted in the next sync.");
Expand All @@ -121,9 +134,10 @@ public void run(ProgressStatus progressStatus) {
e.printStackTrace();
return;
}

progressStatus.increase();
// BUILD COMPLETE LOCAL CHANGELOG

progressStatus.setDescription("Coordinated Sync: Building Local Changelog");
progressStatus.cancelIfCanceled();
Changelog<String, Element> persistedLocalChangelog = new Changelog<>();
Collection<SyncElement> persistedLocalSyncElements = SyncElements.getAllByType(project, SyncElement.Type.LOCAL);
for (SyncElement syncElement : persistedLocalSyncElements) {
Expand Down Expand Up @@ -154,8 +168,10 @@ else if (value instanceof Element) {
localUpdated = localChangelog.get(Changelog.ChangeType.UPDATED),
localDeleted = localChangelog.get(Changelog.ChangeType.DELETED);

progressStatus.increase();
// BUILD COMPLETE MMS CHANGELOG

progressStatus.setDescription("Coordinated Sync: Building Server Changelog");
progressStatus.cancelIfCanceled();
Changelog<String, Void> persistedMmsChangelog = new Changelog<>();
Collection<SyncElement> persistedMmsSyncElements = SyncElements.getAllByType(project, SyncElement.Type.MMS);
for (SyncElement syncElement : persistedMmsSyncElements) {
Expand All @@ -177,9 +193,9 @@ else if (value instanceof Element) {
Map<String, ObjectNode> mmsJsons = new HashMap<>(elementIdsToGet.size());

// Get latest json for element added/changed from MMS

progressStatus.increase();
if (!elementIdsToGet.isEmpty()) {
progressStatus.setDescription("Getting " + elementIdsToGet.size() + " added/changed element" + (elementIdsToGet.size() != 1 ? "s" : "") + " from MMS");
progressStatus.setDescription("Coordinated Sync: Getting " + elementIdsToGet.size() + " added/changed element" + (elementIdsToGet.size() != 1 ? "s" : "") + " from MMS");
File responseFile;
ObjectNode response;
try {
Expand Down Expand Up @@ -220,8 +236,8 @@ else if (value instanceof Element) {
}

// NEW CONFLICT DETECTION

progressStatus.setDescription("Detecting conflicts");
progressStatus.increase();
progressStatus.setDescription("Coordinated Sync: Detecting conflicts");
Map<String, Pair<Changelog.Change<Element>, Changelog.Change<Void>>> conflictedChanges = new LinkedHashMap<>(),
unconflictedChanges = new LinkedHashMap<>();
localChangelog.findConflicts(mmsChangelog, (change, change2) -> change != null && change2 != null, conflictedChanges, unconflictedChanges);
Expand Down Expand Up @@ -305,13 +321,13 @@ else if (shouldUpdate && mmsChange != null) {
}

// POINT OF NO RETURN

progressStatus.increase();
// COMMIT UNCONFLICTED CREATIONS AND UPDATES TO MMS
String projectId = Converters.getIProjectToIdConverter().apply(project.getPrimaryProject());
String refId = MDUtils.getBranchId(project);
boolean shouldLogNoLocalChanges = shouldCommit;
if (shouldCommit && !localElementsToPost.isEmpty()) {
progressStatus.setDescription("Committing creations and updates to MMS");
progressStatus.setDescription("Coordinated Sync: Committing creations and updates to MMS");
LinkedList<ObjectNode> postElements = new LinkedList<>();
for (Element element : localElementsToPost.values()) {
ObjectNode elementObjectNode = Converters.getElementToJsonConverter().apply(element, project);
Expand Down Expand Up @@ -345,12 +361,12 @@ else if (shouldUpdate && mmsChange != null) {
shouldLogNoLocalChanges = false;
}
}

progressStatus.increase();
// COMMIT UNCONFLICTED DELETIONS TO MMS
// NEEDS TO BE AFTER LOCAL; EX: MOVE ELEMENT OUT ON MMS, DELETE OWNER LOCALLY, WHAT HAPPENS?

if (shouldCommit && shouldCommitDeletes && !deleteElements.isEmpty()) {
progressStatus.setDescription("Committing deletions to MMS");
progressStatus.setDescription("Coordinated Sync: Committing deletions to MMS");
try {
File file = MMSUtils.createEntityFile(this.getClass(), ContentType.APPLICATION_JSON, deleteElements, MMSUtils.JsonBlobType.ELEMENT_ID);
HttpRequestBase elementsDeleteRequest = MMSUtils.prepareEndpointBuilderBasicJsonDeleteRequest(MMSElementsEndpoint.builder(), project, file)
Expand All @@ -373,15 +389,15 @@ else if (shouldUpdate && mmsChange != null) {
}

// OUTPUT RESULT OF LOCAL CHANGES

progressStatus.increase();
if (shouldLogNoLocalChanges) {
Application.getInstance().getGUILog().log("[INFO] No local changes to commit to MMS.");
}

// ADD CREATED ELEMENTS LOCALLY FROM MMS
// CHANGE UPDATED ELEMENTS LOCALLY FROM MMS
// REMOVE DELETED ELEMENTS LOCALLY FROM MMS

progressStatus.increase();
if (shouldUpdate) {
listener.setDisabled(true);

Expand All @@ -401,8 +417,8 @@ else if (shouldUpdate && mmsChange != null) {
}

// HANDLE CONFLICTS

progressStatus.setDescription("Finishing up");
progressStatus.increase();
progressStatus.setDescription("Coordinated Sync: Finishing up");

Set<Element> localConflictedElements = new HashSet<>();
Set<ObjectNode> mmsConflictedElements = new HashSet<>();
Expand All @@ -422,7 +438,7 @@ else if (shouldUpdate && mmsChange != null) {
}

ElementValidator elementValidator = new ElementValidator("CSync Conflict Validation", ElementValidator.buildElementPairs(localConflictedElements, project), mmsConflictedElements, project);
elementValidator.run(progressStatus);
ProgressStatusRunner.runWithProgressStatus(elementValidator, "Element Validation", true, 0);
if (!elementValidator.getInvalidElements().isEmpty()) {
Application.getInstance().getGUILog().log("[INFO] There are potential conflicts in " + elementValidator.getInvalidElements().size() + " element" + (elementValidator.getInvalidElements().size() != 1 ? "s" : "") + " between MMS and local changes. Please resolve them and re-sync.");
vss.add(elementValidator.getValidationSuite());
Expand All @@ -447,7 +463,7 @@ else if (shouldUpdate && mmsChange != null) {
}

// CLEAR IN-MEMORY AND PERSIST UNPROCESSED & FAILURES

progressStatus.increase();
listener.setDisabled(true);
Project project = Application.getInstance().getProject();
if (!SessionManager.getInstance().isSessionCreated(project)) {
Expand Down Expand Up @@ -502,7 +518,7 @@ else if (shouldUpdate && mmsChange != null) {

SessionManager.getInstance().closeSession(project);
listener.setDisabled(false);

progressStatus.increase();
// SUCCESS
failure = false;
}
Expand Down
7 changes: 4 additions & 3 deletions src/main/java/org/openmbee/mdk/mms/sync/manual/ManualSyncRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ public void run(ProgressStatus progressStatus) {
progressStatus.setIndeterminate(true);

ProjectValidator pv = new ProjectValidator(project);
pv.validate();

pv.run(progressStatus);
if (pv.hasErrors()) {
Application.getInstance().getGUILog().log("[ERROR] Project validation could not be completed. Manual validation aborted.");
return;
Expand All @@ -64,8 +65,8 @@ public void run(ProgressStatus progressStatus) {
}

if (project.isRemote()) {
BranchValidator bv = new BranchValidator(project);
bv.validate(null, false);
BranchValidator bv = new BranchValidator(project, false);
bv.run(null);
if (bv.hasErrors()) {
Application.getInstance().getGUILog().log("[ERROR] Branch validation could not be completed. Manual validation aborted.");
return;
Expand Down
Loading