Skip to content

Commit

Permalink
Merge branch 'youtrack-7.0-hub-2.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
@penemue
penemue committed Jul 22, 2016
2 parents 8f4e82b + 20ce377 commit 385e393
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 32 deletions.
54 changes: 34 additions & 20 deletions environment/src/main/java/jetbrains/exodus/log/Log.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ public final class Log implements Closeable {
final LogCache cache;

private int logIdentity;
@SuppressWarnings("NullableProblems")
@NotNull
private TransactionalDataWriter bufferedWriter;
/**
Expand Down Expand Up @@ -356,7 +357,7 @@ ArrayByteIterable getHighPage(long alignedAddress) {
return bufferedWriter.getHighPage(alignedAddress);
}

public final int getCachePageSize() {
final int getCachePageSize() {
return cachePageSize;
}

Expand Down Expand Up @@ -614,7 +615,7 @@ public void removeFile(final long address, @NotNull final RemoveBlockType rbt) {
}
}

public void truncateFile(final long address, final long length) {
private void truncateFile(final long address, final long length) {
// truncate physical file
reader.truncateBlock(address, length);
// clear cache
Expand All @@ -632,8 +633,11 @@ public void truncateFile(final long address, final long length) {
* loggable can begin in one file and end in another. Also, this simplifies reading algorithm:
* if we started reading by address it definitely should finish within current file.
*/
public void padWithNulls() {
void padWithNulls() {
long bytesToWrite = fileLengthBound - getLastFileLength();
if (bytesToWrite == 0L) {
throw new ExodusException("Nothing to pad");
}
if (bytesToWrite >= cachePageSize) {
final ArrayByteIterable cachedTailPage = LogCache.getCachedTailPage(cachePageSize);
if (cachedTailPage != null) {
Expand All @@ -645,8 +649,13 @@ public void padWithNulls() {
} while (bytesToWrite >= cachePageSize);
}
}
while (bytesToWrite-- > 0) {
writeContinuously(NullLoggable.create());
if (bytesToWrite == 0) {
bufferedWriter.commit();
createNewFileIfNecessary();
} else {
while (bytesToWrite-- > 0) {
writeContinuously(NullLoggable.create());
}
}
}

Expand Down Expand Up @@ -808,21 +817,7 @@ public long writeContinuously(final Loggable loggable) {
}
bufferedWriter.commit();
highAddress += recordLength;
if (getLastFileLength() == 0 || System.currentTimeMillis() > lastSyncTicks + config.getSyncPeriod()) {
flush(true);
if (getLastFileLength() == 0) {
bufferedWriter.close();
if (config.isFullFileReadonly()) {
final Long lastFile;
synchronized (blockAddrs) {
lastFile = blockAddrs.getMaximum();
}
if (lastFile != null) {
reader.getBlock(lastFile).setWritable(false);
}
}
}
}
createNewFileIfNecessary();
return result;
} catch (Throwable e) {
highAddress = result;
Expand All @@ -834,6 +829,25 @@ public long writeContinuously(final Loggable loggable) {
}
}

private void createNewFileIfNecessary() {
final boolean shouldCreateNewFile = getLastFileLength() == 0;
if (shouldCreateNewFile) {
flush(true);
bufferedWriter.close();
if (config.isFullFileReadonly()) {
final Long lastFile;
synchronized (blockAddrs) {
lastFile = blockAddrs.getMaximum();
}
if (lastFile != null) {
reader.getBlock(lastFile).setWritable(false);
}
}
} else if (System.currentTimeMillis() > lastSyncTicks + config.getSyncPeriod()) {
flush(true);
}
}

/**
* Sets LogTestConfig.
* Is destined for tests only, please don't set a not-null value in application code.
Expand Down
67 changes: 55 additions & 12 deletions environment/src/test/java/jetbrains/exodus/tree/TreePutTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,12 @@
package jetbrains.exodus.tree;

import jetbrains.exodus.ByteIterable;
import jetbrains.exodus.TestFor;
import jetbrains.exodus.TestUtil;
import jetbrains.exodus.bindings.IntegerBinding;
import jetbrains.exodus.bindings.LongBinding;
import jetbrains.exodus.core.dataStructures.hash.IntHashMap;
import jetbrains.exodus.core.dataStructures.hash.IntHashSet;
import org.junit.Assert;
import org.junit.Test;

Expand Down Expand Up @@ -312,7 +314,7 @@ public void testPutRandomWithoutDuplicates() throws Throwable {
final IntHashMap<String> map = new IntHashMap<>();
final int count = 200000;

TestUtil.time("Put took ", new Runnable() {
TestUtil.time("put()", new Runnable() {
@Override
public void run() {
for (int i = 0; i < count; ++i) {
Expand All @@ -325,7 +327,7 @@ public void run() {
});

Assert.assertEquals(map.size(), tm.getSize());
TestUtil.time("Get took ", new Runnable() {
TestUtil.time("get()", new Runnable() {
@Override
public void run() {
for (final Map.Entry<Integer, String> entry : map.entrySet()) {
Expand All @@ -344,7 +346,7 @@ public void testPutRandomWithoutDuplicates2() throws Throwable {
final IntHashMap<String> map = new IntHashMap<>();
final int count = 200000;

TestUtil.time("Put took ", new Runnable() {
TestUtil.time("put()", new Runnable() {
@Override
public void run() {
for (int i = 0; i < count; ++i) {
Expand All @@ -362,7 +364,7 @@ public void run() {

Assert.assertEquals(map.size(), t.getSize());

TestUtil.time("Get took ", new Runnable() {
TestUtil.time("get()", new Runnable() {
@Override
public void run() {
for (final Map.Entry<Integer, String> entry : map.entrySet()) {
Expand All @@ -379,14 +381,14 @@ public void testPutRightRandomWithoutDuplicates() throws Throwable {
tm = createMutableTree(false, 1);

final IntHashMap<String> map = new IntHashMap<>();
final int count = 200000;
final int count = 99999;

TestUtil.time("PutRight took ", new Runnable() {
TestUtil.time("putRight()", new Runnable() {
@Override
public void run() {
for (int i = 0; i < count; ++i) {
final String value = Integer.toString(i);
tm.put(key(Integer.toString(i)), value(value));
tm.putRight(key(i), value(value));
map.put(i, value);
}
}
Expand All @@ -398,13 +400,13 @@ public void run() {

Assert.assertEquals(map.size(), t.getSize());

TestUtil.time("Get took ", new Runnable() {
TestUtil.time("get()", new Runnable() {
@Override
public void run() {
for (final Map.Entry<Integer, String> entry : map.entrySet()) {
final Integer key = entry.getKey();
final String value = entry.getValue();
valueEquals(value, t.get(key(Integer.toString(key))));
valueEquals(value, t.get(key(key)));
}
}
});
Expand All @@ -417,7 +419,7 @@ public void testAddRandomWithoutDuplicates() throws Throwable {
final IntHashMap<String> map = new IntHashMap<>();
final int count = 50000;

TestUtil.time("Add took ", new Runnable() {
TestUtil.time("add()", new Runnable() {
@Override
public void run() {
for (int i = 0; i < count; ++i) {
Expand All @@ -437,7 +439,7 @@ public void run() {

Assert.assertEquals(map.size(), t.getSize());

TestUtil.time("Get took ", new Runnable() {
TestUtil.time("get()", new Runnable() {
@Override
public void run() {
for (final Map.Entry<Integer, String> entry : map.entrySet()) {
Expand All @@ -450,7 +452,7 @@ public void run() {

tm = t.getMutableCopy();

TestUtil.time("Failing add took ", new Runnable() {
TestUtil.time("Failing add()", new Runnable() {
@Override
public void run() {
for (final Map.Entry<Integer, String> entry : map.entrySet()) {
Expand All @@ -461,4 +463,45 @@ public void run() {
}
});
}

@Test
@TestFor(issues = "XD-539")
public void createHugeTree() throws Throwable {
if (Runtime.getRuntime().maxMemory() < 4000000000L) {
return;
}

tm = createMutableTree(false, 1);

final IntHashSet set = new IntHashSet();
final int count = 20000;
final StringBuilder builder = new StringBuilder("value");

TestUtil.time("put()", new Runnable() {
@Override
public void run() {
for (int i = 0; i < count; ++i) {
tm.put(key(Integer.toString(i)), value(builder.toString()));
set.add(i);
builder.append(i);
}
}
});

final long address = tm.save();
System.out.println("Log size: " + tm.getLog().getHighAddress());
reopen();
t = openTree(address, false);

Assert.assertEquals(set.size(), t.getSize());

TestUtil.time("get()", new Runnable() {
@Override
public void run() {
for (Integer i : set) {
assertTrue(t.hasKey(key(Integer.toString(i))));
}
}
});
}
}

0 comments on commit 385e393

Please sign in to comment.