Skip to content

Commit

Permalink
Huge performance increase when listing artifacts, e.g. Jenkins job or…
Browse files Browse the repository at this point in the history 
… run screens (#21)
  • Loading branch information
@timja
timja authored May 19, 2021
1 parent 2c3e4a2 commit 5205e91
Show file tree
Hide file tree
Showing 7 changed files with 202 additions and 56 deletions.
3 changes: 1 addition & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
<changelist>9999-SNAPSHOT</changelist>
<jenkins.version>2.277.2</jenkins.version>
<java.level>8</java.level>
<windows-azure-storage.version>356.vfa287acbfe71</windows-azure-storage.version>
</properties>

<licenses>
Expand Down Expand Up @@ -56,7 +55,7 @@
<dependency>
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>windows-azure-storage</artifactId>
<version>${windows-azure-storage.version}</version>
<version>357.v36c07129d7d3</version>
</dependency>
<dependency>
<groupId>org.jenkins-ci.plugins.workflow</groupId>
Expand Down
6 changes: 4 additions & 2 deletions src/main/java/com/microsoft/jenkins/artifactmanager/AzureArtifactManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ public void archive(FilePath workspace, Launcher launcher, BuildListener listene
.act(new ContentTypeGuesser(new ArrayList<>(artifacts.keySet()), listener));

try {
BlobContainerClient container = Utils.getBlobContainerReference(accountInfo, config.getContainer(), false);
BlobContainerClient container = Utils.getBlobContainerReference(accountInfo, config.getContainer(), true);

for (Map.Entry<String, String> entry : contentTypes.entrySet()) {
String path = "artifacts/" + entry.getKey();
Expand Down Expand Up @@ -339,7 +339,7 @@ private BlobContainerClient getContainer() throws IOException,
return Utils.getBlobContainerReference(
accountInfo,
this.actualContainerName,
true
false
);
}

Expand Down Expand Up @@ -373,6 +373,7 @@ public void stash(@Nonnull String name, @Nonnull FilePath workspace, @Nonnull La
serviceData.setFilePath(stashTempFile.getName());
serviceData.setUploadType(UploadType.INDIVIDUAL);

// TODO rewrite to not use azure-storage plugin API like in artifacts
UploadService uploadService = new UploadToBlobService(serviceData);
try {
uploadService.execute();
Expand Down Expand Up @@ -401,6 +402,7 @@ public void unstash(@Nonnull String name, @Nonnull FilePath workspace, @Nonnull
serviceData.setIncludeFilesPattern(stashes + name + Constants.TGZ_FILE_EXTENSION);
serviceData.setFlattenDirectories(true);

workspace.mkdirs();
DownloadService downloadService = new DownloadFromContainerService(serviceData);
try {
downloadService.execute();
Expand Down
194 changes: 165 additions & 29 deletions src/main/java/com/microsoft/jenkins/artifactmanager/AzureBlobVirtualFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,32 +9,34 @@
import com.azure.storage.blob.BlobClient;
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.models.BlobItem;
import com.azure.storage.blob.models.BlobItemProperties;
import com.azure.storage.blob.models.BlobProperties;
import com.azure.storage.blob.models.BlobStorageException;
import com.azure.storage.blob.models.ListBlobsOptions;
import com.microsoftopentechnologies.windowsazurestorage.beans.StorageAccountInfo;
import edu.umd.cs.findbugs.annotations.CheckForNull;
import edu.umd.cs.findbugs.annotations.NonNull;
import hudson.model.Run;
import hudson.remoting.Callable;
import jenkins.util.VirtualFile;
import org.apache.commons.lang.StringUtils;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;

Expand All @@ -43,7 +45,6 @@ public class AzureBlobVirtualFile extends AzureAbstractVirtualFile {
private static final long serialVersionUID = 9054620703341308471L;

private static final Logger LOGGER = Logger.getLogger(AzureBlobVirtualFile.class.getName());
private static final String AZURE_BLOB_URL_PATTERN = "https://%s.blob.core.windows.net/%s/%s";
private static final int NOT_FOUND = 404;

private final String container;
Expand All @@ -64,11 +65,88 @@ public String getKey() {
return this.key;
}

/**
* Cache of metadata collected during {@link #run}.
* Keys are {@link #container}.
* Values are a stack of cache frames, one per nested {@link #run} call.
*/
private static final ThreadLocal<Map<String, Deque<CacheFrame>>> CACHE = ThreadLocal.withInitial(HashMap::new);

private static final class CacheFrame {
/** {@link #key} of the root virtual file plus a trailing {@code /}. */
private final String root;
/**
* Information about all known (recursive) child <em>files</em> (not directories).
* Keys are {@code /}-separated relative paths.
* If the root itself happened to be a file, that information is not cached.
*/
private final Map<String, CachedMetadata> children;
CacheFrame(String root, Map<String, CachedMetadata> children) {
this.root = root;
this.children = children;
}
}

/**
* Record that a given file exists.
*/
private static final class CachedMetadata {
private final long length, lastModified;
CachedMetadata(long length, long lastModified) {
this.length = length;
this.lastModified = lastModified;
}
}


@Override
public <V> V run(Callable<V, IOException> callable) throws IOException {
return super.run(callable);
LOGGER.log(Level.FINE, "enter cache {0} / {1}", new Object[] {container, key});
Deque<CacheFrame> stack = cacheFrames();
Map<String, CachedMetadata> saved = new HashMap<>();
try {
StorageAccountInfo accountInfo = Utils.getStorageAccount(build.getParent());
BlobContainerClient blobContainerReference = Utils.getBlobContainerReference(accountInfo, this.container,
false);
ListBlobsOptions listBlobsOptions = new ListBlobsOptions()
.setPrefix(key);

for (BlobItem sm : blobContainerReference.listBlobs(listBlobsOptions, null)) {
BlobItemProperties properties = sm.getProperties();
OffsetDateTime lastModified = properties.getLastModified();
long lastModifiedMilli = lastModified.toInstant().toEpochMilli();
String fileName = stripBeginningSlash(sm.getName().replaceFirst(key, ""));
saved.put(fileName,
new CachedMetadata(properties.getContentLength(), lastModifiedMilli));
}
} catch (RuntimeException | URISyntaxException x) {
throw new IOException(x);
}
stack.push(new CacheFrame(stripTrailingSlash(key) + "/", saved));
try {
LOGGER.log(Level.FINE, "using cache {0} / {1}: {2} file entries",
new Object[] {container, key, saved.size()});
return callable.call();
} finally {
LOGGER.log(Level.FINE, "exit cache {0} / {1}", new Object[] {container, key});
stack.pop();
}
}

private Deque<CacheFrame> cacheFrames() {
return CACHE.get().computeIfAbsent(container, c -> new ArrayDeque<>());
}

/**
* Finds a cache frame whose {@link CacheFrame#root} is a prefix of the given {@link #key}
* or {@code /}-appended variant.
*
*/
private @CheckForNull CacheFrame findCacheFrame(String cacheKey) {
return cacheFrames().stream().filter(frame -> cacheKey.startsWith(frame.root)).findFirst().orElse(null);
}


@NonNull
@Override
public String getName() {
Expand All @@ -85,27 +163,21 @@ private String stripTrailingSlash(String string) {
return localKey;
}

private String stripBeginningSlash(String string) {
String localKey = string;
if (string.startsWith("/")) {
localKey = localKey.substring(1);
}
return localKey;
}

@NonNull
@Override
public URI toURI() {
StorageAccountInfo accountInfo = Utils.getStorageAccount(build.getParent());
try {
String encodedKey = StringUtils.replace(this.key, "%", "%25");
String formattedUrl = String.format(AZURE_BLOB_URL_PATTERN, accountInfo.getStorageAccName(),
container, encodedKey);

String decodedURL = URLDecoder.decode(formattedUrl, StandardCharsets.UTF_8.name());
URL url = new URL(decodedURL);
return new URI(
url.getProtocol(),
url.getUserInfo(),
url.getHost(),
url.getPort(),
url.getPath(),
url.getQuery(),
url.getRef()
);
} catch (URISyntaxException | MalformedURLException | UnsupportedEncodingException e) {
return new URI(Utils.getBlobUrl(accountInfo, this.container, this.key));
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}
Expand All @@ -131,14 +203,34 @@ public VirtualFile getParent() {

@Override
public boolean isDirectory() throws IOException {
String keyWithSlash = stripTrailingSlash(this.key) + Constants.FORWARD_SLASH;
LOGGER.log(Level.FINE, "checking directory status {0} / {1}", new Object[]{container, keyWithSlash});
String keyWithNoSlash = stripTrailingSlash(this.key);

if (keyWithNoSlash.endsWith("/*view*")) {
return false;
}

String keyS = keyWithNoSlash + "/";
CacheFrame frame = findCacheFrame(keyS);
if (frame != null) {
LOGGER.log(Level.FINER, "cache hit on directory status of {0} / {1}", new Object[] {container, this.key});
String relSlash = stripTrailingSlash(keyS.substring(frame.root.length())); // "" or "sub/dir/"
Set<String> children = frame.children.keySet();
boolean existsInCache = children.stream().anyMatch(f -> f.startsWith(relSlash));
// if we don't know about it then it's not a directory
if (!existsInCache) {
return false;
}
// if it's not an exact file path then it's not a directory
return children.stream().noneMatch(f -> f.equals(relSlash));
}

LOGGER.log(Level.FINE, "checking directory status {0} / {1}", new Object[]{container, keyWithNoSlash});

StorageAccountInfo accountInfo = Utils.getStorageAccount(build.getParent());
try {
BlobContainerClient blobContainerReference = Utils.getBlobContainerReference(accountInfo, this.container,
false);
Iterator<BlobItem> iterator = blobContainerReference.listBlobsByHierarchy(keyWithSlash).iterator();
Iterator<BlobItem> iterator = blobContainerReference.listBlobsByHierarchy(keyS).iterator();
return iterator.hasNext();
} catch (URISyntaxException e) {
throw new IOException(e);
Expand All @@ -147,6 +239,20 @@ public boolean isDirectory() throws IOException {

@Override
public boolean isFile() throws IOException {
String keyS = key + "/";

if (keyS.endsWith("/*view*/")) {
return false;
}

CacheFrame frame = findCacheFrame(keyS);
if (frame != null) {
String rel = stripTrailingSlash(keyS.substring(frame.root.length()));
CachedMetadata metadata = frame.children.get(rel);
LOGGER.log(Level.FINER, "cache hit on file status of {0} / {1}", new Object[] {container, key});
return metadata != null;
}

StorageAccountInfo accountInfo = Utils.getStorageAccount(build.getParent());
try {
BlobContainerClient blobContainerReference = Utils.getBlobContainerReference(accountInfo, this.container,
Expand All @@ -166,10 +272,22 @@ public boolean exists() throws IOException {
@NonNull
@Override
public VirtualFile[] list() throws IOException {
if (StringUtils.isBlank(this.container)) {
// compatible with version before 0.1.2
return new VirtualFile[0];
String keyS = key + "/";
CacheFrame frame = findCacheFrame(keyS);
if (frame != null) {
LOGGER.log(Level.FINER, "cache hit on listing of {0} / {1}", new Object[] {container, key});
String relSlash = keyS.substring(frame.root.length()); // "" or "sub/dir/"
VirtualFile[] virtualFiles = frame.children.keySet().stream() // filenames relative to frame root
.filter(f -> f.startsWith(relSlash)) // those inside this dir
// just the file simple name, or direct subdir name
.map(f -> f.substring(relSlash.length()).replaceFirst("/.+", ""))
.distinct() // ignore duplicates if have multiple files under one direct subdir
// direct children
.map(simple -> new AzureBlobVirtualFile(this.container, keyS + simple, this.build))
.toArray(VirtualFile[]::new);
return virtualFiles;
}

VirtualFile[] list;
String keys = stripTrailingSlash(this.key) + Constants.FORWARD_SLASH;

Expand Down Expand Up @@ -202,6 +320,15 @@ public VirtualFile child(@NonNull String name) {

@Override
public long length() throws IOException {
String keyS = key + "/";
CacheFrame frame = findCacheFrame(keyS);
if (frame != null) {
String rel = stripTrailingSlash(keyS.substring(frame.root.length()));
CachedMetadata metadata = frame.children.get(rel);
LOGGER.log(Level.FINER, "cache hit on length of {0} / {1}", new Object[] {container, key});
return metadata != null ? metadata.length : 0;
}

StorageAccountInfo accountInfo = Utils.getStorageAccount(build.getParent());
try {
BlobContainerClient blobContainerReference = Utils.getBlobContainerReference(accountInfo, this.container,
Expand All @@ -223,6 +350,15 @@ public long length() throws IOException {

@Override
public long lastModified() throws IOException {
String keyS = key + "/";
CacheFrame frame = findCacheFrame(keyS);
if (frame != null) {
String rel = stripTrailingSlash(keyS.substring(frame.root.length()));
CachedMetadata metadata = frame.children.get(rel);
LOGGER.log(Level.FINER, "cache hit on lastModified of {0} / {1}", new Object[] {container, key});
return metadata != null ? metadata.lastModified : 0;
}

if (isDirectory()) {
return 0;
}
Expand Down
Loading

0 comments on commit 5205e91

Please sign in to comment.