Skip to content

Commit

Permalink
Align FileBlobStore with parameter handling practice of S3/Azure
Browse files Browse the repository at this point in the history
  • Loading branch information
@aaime
aaime committed May 13, 2024
1 parent 5666d40 commit ce457f9
Show file tree
Hide file tree
Showing 2 changed files with 130 additions and 41 deletions.
152 changes: 111 additions & 41 deletions geowebcache/core/src/main/java/org/geowebcache/storage/blobstore/file/FileBlobStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,19 @@
*/
package org.geowebcache.storage.blobstore.file;

import static java.util.Objects.isNull;
import static org.geowebcache.storage.blobstore.file.FilePathUtils.filteredGridSetId;
import static org.geowebcache.storage.blobstore.file.FilePathUtils.filteredLayerName;
import static org.geowebcache.util.FileUtils.listFilesNullSafe;
import static org.geowebcache.util.TMSKeyBuilder.PARAMETERS_METADATA_OBJECT_PREFIX;
import static org.geowebcache.util.TMSKeyBuilder.PARAMETERS_METADATA_OBJECT_SUFFIX;

import com.google.common.base.Preconditions;
import java.io.File;
import java.io.FileFilter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
Expand All @@ -34,6 +38,7 @@
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
Expand Down Expand Up @@ -66,6 +71,10 @@
/** See BlobStore interface description for details */
public class FileBlobStore implements BlobStore {

interface Writer {
void write(File file) throws IOException;
}

private static Logger log =
Logging.getLogger(org.geowebcache.storage.blobstore.file.FileBlobStore.class.getName());

Expand Down Expand Up @@ -372,7 +381,7 @@ private File getLayerPath(String layerName) {
/** Delete a particular tile */
@Override
public boolean delete(TileObject stObj) throws StorageException {
File fh = getFileHandleTile(stObj, null);
File fh = getFileHandleTile(stObj, false);
boolean ret = false;
// we call fh.length() here to check whether the file exists and its length in a single
// operation cause lots of calls to exists() may raise the file system cache usage to the
Expand Down Expand Up @@ -466,7 +475,7 @@ public void postVisitDirectory(File dir) {
*/
@Override
public boolean get(TileObject stObj) throws StorageException {
File fh = getFileHandleTile(stObj, null);
File fh = getFileHandleTile(stObj, false);
if (!fh.exists()) {
stObj.setStatus(Status.MISS);
return false;
Expand All @@ -482,17 +491,11 @@ public boolean get(TileObject stObj) throws StorageException {
/** Store a tile. */
@Override
public void put(TileObject stObj) throws StorageException {

// an update to ParameterMap file is required !!!
Runnable upm =
() -> {
this.persistParameterMap(stObj);
};
final File fh = getFileHandleTile(stObj, upm);
final File fh = getFileHandleTile(stObj, true);
final long oldSize = fh.length();
final boolean existed = oldSize > 0;

writeFile(fh, stObj, existed);
writeTile(fh, stObj, existed);

// mark the last modification as the tile creation time if set, otherwise
// we'll leave it to the writing time
Expand All @@ -507,6 +510,8 @@ public void put(TileObject stObj) throws StorageException {
}
}

putParametersMetadata(stObj.getLayerName(), stObj.getParametersId(), stObj.getParameters());

/*
* this is important because listeners may be tracking tile existence
*/
Expand All @@ -518,8 +523,40 @@ public void put(TileObject stObj) throws StorageException {
}
}

private File getFileHandleTile(TileObject stObj, Runnable onTileFolderCreation)
private void putParametersMetadata(
String layerName, String parametersId, Map<String, String> parameters)
throws StorageException {
assert (isNull(parametersId) == isNull(parameters));
if (isNull(parametersId)) {
return;
}
File parametersFile = parametersFile(layerName, parametersId);
if (parametersFile.exists()) return;

writeFile(
parametersFile,
false,
file -> {
Properties properties = new Properties();
parameters.forEach(properties::setProperty);
try (OutputStream os = new FileOutputStream(file)) {
properties.store(os, "Parameters values for identifier: " + parametersId);
}
});
}

private File parametersFile(String layerName, String parametersId) {
String path =
FilePathUtils.buildPath(
this.path,
layerName,
PARAMETERS_METADATA_OBJECT_PREFIX
+ parametersId
+ PARAMETERS_METADATA_OBJECT_SUFFIX);
return new File(path);
}

private File getFileHandleTile(TileObject stObj, boolean createParent) throws StorageException {
final MimeType mimeType;
try {
mimeType = MimeType.createFromFormat(stObj.getBlobFormat());
Expand All @@ -535,12 +572,10 @@ private File getFileHandleTile(TileObject stObj, Runnable onTileFolderCreation)
throw new StorageException("Failed to compute file path", e);
}

// check if it's required to create tile folder
if (onTileFolderCreation != null) {
if (createParent) {
log.fine("Creating parent tile folder and updating ParameterMap");
File parent = tilePath.getParentFile();
mkdirs(parent, stObj);
onTileFolderCreation.run();
}

return tilePath;
Expand All @@ -553,16 +588,32 @@ private Resource readFile(File fh) throws StorageException {
return new FileResource(fh);
}

private void writeFile(File target, TileObject stObj, boolean existed) throws StorageException {
private void writeTile(File target, TileObject stObj, boolean existed) throws StorageException {
writeFile(
target,
existed,
file -> {
try (FileOutputStream fos = new FileOutputStream(file);
FileChannel channel = fos.getChannel()) {
stObj.getBlob().transferTo(channel);
}
});
}

/**
* Writes into the target file by first creating a temporary file, filling it with the writer,
* and then renaming it to the target file.
*
* @throws StorageException
*/
private void writeFile(File target, boolean existed, Writer writer) throws StorageException {
// first write to temp file
tmp.mkdirs();
File temp = new File(tmp, tmpGenerator.newName());

try {
// open the output stream and read the blob into the tile
try (FileOutputStream fos = new FileOutputStream(temp);
FileChannel channel = fos.getChannel()) {
stObj.getBlob().transferTo(channel);
try {
writer.write(temp);
} catch (IOException ioe) {
throw new StorageException(ioe.getMessage() + " for " + target.getAbsolutePath());
}
Expand All @@ -578,9 +629,7 @@ private void writeFile(File target, TileObject stObj, boolean existed) throws St
temp = null;
}
}

} finally {

if (temp != null) {
log.warning(
"Tile "
Expand All @@ -591,15 +640,6 @@ private void writeFile(File target, TileObject stObj, boolean existed) throws St
}
}

protected void persistParameterMap(TileObject stObj) {
if (Objects.nonNull(stObj.getParametersId())) {
putLayerMetadata(
stObj.getLayerName(),
"parameters." + stObj.getParametersId(),
ParametersUtils.getKvp(stObj.getParameters()));
}
}

@Override
public void clear() throws StorageException {
throw new StorageException("Not implemented yet!");
Expand Down Expand Up @@ -726,6 +766,10 @@ public boolean deleteByParametersId(String layerName, String parametersId)
return false;
}

// delete the parameter file
parametersFile(layerName, parametersId).delete();

// delete the caches
File[] parameterCaches =
listFilesNullSafe(
layerPath,
Expand Down Expand Up @@ -779,25 +823,51 @@ public boolean isParameterIdCached(String layerName, final String parametersId)

@Override
public Map<String, Optional<Map<String, String>>> getParametersMapping(String layerName) {
Set<String> parameterIds = getParameterIds(layerName);

// for backwards compatibility, check the parameters in the metadata file
Map<String, String> p;
try {
p = layerMetadata.getLayerMetadata(layerName);
} catch (IOException e) {
log.fine("Optimistic read of metadata mappings failed");
return null;
}
return getParameterIds(layerName).stream()
Map<String, Optional<Map<String, String>>> result =
parameterIds.stream()
.collect(
Collectors.toMap(
(id) -> id,
(id) -> {
String kvp = p.get("parameters." + id);
if (Objects.isNull(kvp)) {
return Optional.empty();
}
kvp = urlDecUtf8(kvp);
return Optional.of(ParametersUtils.getMap(kvp));
}));

// go look for the current parameter files too though, and overwrite the legacy metadata
for (String parameterId : parameterIds) {
File file = parametersFile(layerName, parameterId);
if (file.exists()) {
try {
Properties properties = new Properties();
properties.load(Files.newInputStream(file.toPath()));
result.put(parameterId, Optional.of(propertiesToMap(properties)));
} catch (IOException e) {
throw new RuntimeException("Failed to read parameters file", e);
}
}
}

return result;
}

private static Map<String, String> propertiesToMap(Properties properties) {
return properties.entrySet().stream()
.collect(
Collectors.toMap(
(id) -> id,
(id) -> {
String kvp = p.get("parameters." + id);
if (Objects.isNull(kvp)) {
return Optional.empty();
}
kvp = urlDecUtf8(kvp);
return Optional.of(ParametersUtils.getMap(kvp));
}));
Collectors.toMap(e -> e.getKey().toString(), e -> e.getValue().toString()));
}

static final int paramIdLength =
Expand Down
19 changes: 19 additions & 0 deletions geowebcache/core/src/main/java/org/geowebcache/storage/blobstore/file/FilePathUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
*/
package org.geowebcache.storage.blobstore.file;

import java.io.File;

public class FilePathUtils {

public static String gridsetZoomLevelDir(String gridSetId, long zoomLevel) {
Expand Down Expand Up @@ -116,4 +118,21 @@ public static void appendGridsetZoomLevelDir(String gridSetId, long z, StringBui
path.append('_');
zeroPadder(z, 2, path);
}

/**
* Returns a path built from a root and a list of components. The components are appended to the
* root with a {@link File#separatorChar} in between. The root is trusted not to need escaping,
* all other bits are filtered.
*/
public static String buildPath(String root, String... components) {
StringBuilder path = new StringBuilder(256);
path.append(root);

for (String component : components) {
path.append(File.separatorChar);
appendFiltered(component, path);
}

return path.toString();
}
}

0 comments on commit ce457f9

Please sign in to comment.