Skip to content

Commit

Permalink
Merge pull request #4265 from rouault/embed
Browse files Browse the repository at this point in the history
Add CMake options EMBED_RESOURCE_FILES and USE_ONLY_EMBEDDED_RESOURCE_FILES for proj.db and proj.ini embedding
  • Loading branch information
rouault authored Oct 28, 2024
2 parents 4ada7ae + 0a14c19 commit 463d104
Show file tree
Hide file tree
Showing 20 changed files with 1,030 additions and 78 deletions.
4 changes: 4 additions & 0 deletions .github/workflows/clang_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ jobs:
id: regular
PROJ_CMAKE_BUILD_OPTIONS: ""

- name: EMBED_RESOURCE_FILES
id: EMBED_RESOURCE_FILES
PROJ_CMAKE_BUILD_OPTIONS: "-DEMBED_RESOURCE_FILES=ON"

- name: Without TIFF
id: without_tiff
PROJ_CMAKE_BUILD_OPTIONS: "-DENABLE_TIFF=OFF"
Expand Down
37 changes: 37 additions & 0 deletions .github/workflows/fedora_rawhide.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
name: Fedora Rawhide

on:
push:
paths-ignore:
- 'docs/**'
pull_request:
paths-ignore:
- 'docs/**'

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true

permissions:
contents: read

jobs:

fedora_rawhide:
runs-on: ubuntu-latest
if: "!contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]')"
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Cache
uses: actions/cache@v4
id: cache
with:
path: |
${{ github.workspace }}/ccache.tar.gz
key: ${{ runner.os }}-cache-fedora_rawhide-${{ github.run_id }}
restore-keys: ${{ runner.os }}-cache-fedora_rawhide-

- name: Run
run: docker run -e CI -e WORK_DIR="$PWD" -v $PWD:$PWD fedora:rawhide $PWD/.github/workflows/fedora_rawhide/start.sh
31 changes: 31 additions & 0 deletions .github/workflows/fedora_rawhide/start.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/bin/bash

set -e

dnf install -y cmake clang ccache ninja-build sqlite-devel libtiff-devel libcurl-devel diffutils

cd "$WORK_DIR"

if test -f "$WORK_DIR/ccache.tar.gz"; then
echo "Restoring ccache..."
(cd $HOME && tar xzf "$WORK_DIR/ccache.tar.gz")
fi

export CCACHE_CPP2=yes

ccache -M 500M
ccache -s

mkdir build
cd build
CC=clang CXX=clang++ cmake .. \
-DEMBED_RESOURCE_FILES=ON -DUSE_ONLY_EMBEDDED_RESOURCE_FILES=ON -DUSE_CCACHE=ON -DPROJ_DB_CACHE_DIR=$HOME/.ccache ..
make -j$(nproc)
ctest -j$(nproc)
cd ..

ccache -s

echo "Saving ccache..."
rm -f "$WORK_DIR/ccache.tar.gz"
(cd $HOME && tar czf "$WORK_DIR/ccache.tar.gz" .cache)
41 changes: 41 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,45 @@ else()
message(STATUS "Testing disabled")
endif()

################################################################################
# Whether we should embed resources
################################################################################

function (is_sharp_embed_available res)
if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.21 AND
((CMAKE_C_COMPILER_ID STREQUAL "GNU") OR (CMAKE_C_COMPILER_ID STREQUAL "Clang")))
# CMAKE_C_STANDARD=23 only supported since CMake 3.21
set(TEST_SHARP_EMBED
"static const unsigned char embedded[] = {\n#embed __FILE__\n};\nint main() { (void)embedded; return 0;}"
)
set(CMAKE_C_STANDARD_BACKUP "${CMAKE_C_STANDARD}")
set(CMAKE_C_STANDARD "23")
check_c_source_compiles("${TEST_SHARP_EMBED}" _TEST_SHARP_EMBED)
set(CMAKE_C_STANDARD "${CMAKE_C_STANDARD_BACKUP}")
if (_TEST_SHARP_EMBED)
set(${res} ON PARENT_SCOPE)
else()
set(${res} OFF PARENT_SCOPE)
endif()
else()
set(${res} OFF PARENT_SCOPE)
endif()
endfunction()

is_sharp_embed_available(IS_SHARP_EMBED_AVAILABLE_RES)
if (NOT BUILD_SHARED_LIBS)
set(DEFAULT_EMBED_RESOURCE_FILES ON)
else()
set(DEFAULT_EMBED_RESOURCE_FILES OFF)
endif()
option(EMBED_RESOURCE_FILES "Whether resource files (limited to proj.db) should be embedded into the PROJ library" ${DEFAULT_EMBED_RESOURCE_FILES})

option(USE_ONLY_EMBEDDED_RESOURCE_FILES "Whether embedded resource files (limited to proj.db) should be used (should nominally be used together with EMBED_RESOURCE_FILES=ON, otherwise this will result in non-functional builds)" OFF)

if (USE_ONLY_EMBEDDED_RESOURCE_FILES AND NOT EMBED_RESOURCE_FILES)
message(WARNING "USE_ONLY_EMBEDDED_RESOURCE_FILES=ON set but EMBED_RESOURCE_FILES=OFF: some drivers will lack required resource files")
endif()

################################################################################
# Build configured components
################################################################################
Expand Down Expand Up @@ -432,3 +471,5 @@ endif()

configure_file(cmake/uninstall.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/proj_uninstall.cmake @ONLY)
add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/proj_uninstall.cmake)

message(STATUS "EMBED_RESOURCE_FILES=${EMBED_RESOURCE_FILES}")
99 changes: 99 additions & 0 deletions cmake/FileEmbed.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
# Derived from https://gitlab.com/jhamberg/cmake-examples/-/blob/master/cmake/FileEmbed.cmake
# MIT licensed
# Copyright (c) 2022 Jonathan Hamberg

function(FileEmbedSetup)

if (NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/file_embed)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}file_embed)
endif ()

if (NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/file_embed/file_embed_empty.c)
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/file_embed/file_embed_empty.c "")
endif ()

add_library(file_embed ${CMAKE_CURRENT_BINARY_DIR}/file_embed/file_embed_empty.c)
target_include_directories(file_embed PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/file_embed)

endfunction()

function(FileEmbedAdd file)
FileEmbedGenerate(${file} var)
target_sources(file_embed PUBLIC ${var})

add_custom_command(
OUTPUT ${var}
COMMAND ${CMAKE_COMMAND}
-DRUN_FILE_EMBED_GENERATE=1
"-DFILE_EMBED_GENERATE_PATH=${file}"
-P ${PROJECT_SOURCE_DIR}/cmake/FileEmbed.cmake
MAIN_DEPENDENCY ${file}
)
endfunction()

function(FileEmbedGenerate file generated_c)

get_filename_component(base_filename ${file} NAME)
set(output_filename "${base_filename}.c")
string(MAKE_C_IDENTIFIER ${base_filename} c_name)
file(READ ${file} content HEX)

string(LENGTH "${content}" size)
math(EXPR size_mult_16 "${size} - (${size} % 16)")
string(SUBSTRING "${content}" 0 ${size_mult_16} content_mult_16)

string(REGEX REPLACE
"([A-Fa-f0-9][A-Fa-f0-9])\
([A-Fa-f0-9][A-Fa-f0-9])\
([A-Fa-f0-9][A-Fa-f0-9])\
([A-Fa-f0-9][A-Fa-f0-9])\
([A-Fa-f0-9][A-Fa-f0-9])\
([A-Fa-f0-9][A-Fa-f0-9])\
([A-Fa-f0-9][A-Fa-f0-9])\
([A-Fa-f0-9][A-Fa-f0-9])"
"0x\\1,0x\\2,0x\\3,0x\\4,0x\\5,0x\\6,0x\\7,0x\\8,\n" SEPARATED_HEX "${content_mult_16}")

set(SEPARATED_HEX_REMAINDER "")
if (NOT ${size_mult_16} EQUAL ${size})
string(SUBSTRING "${content}" ${size_mult_16} 16 content_remainder)
string(REGEX REPLACE "([A-Fa-f0-9][A-Fa-f0-9])" "0x\\1," SEPARATED_HEX_REMAINDER "${content_remainder}")
endif()

set(output_c "${SEPARATED_HEX}${SEPARATED_HEX_REMAINDER}")

set(output_c "
#include \"${c_name}.h\"
const uint8_t ${c_name}_data[] = {
${output_c}
}\;
const unsigned ${c_name}_size = sizeof(${c_name}_data)\;
")

set(output_h "
#ifndef ${c_name}_H
#define ${c_name}_H
#include <stdint.h>
extern const uint8_t ${c_name}_data[]\;
extern const unsigned ${c_name}_size\;
#endif // ${c_name}_H
")


if (NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/file_embed)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}file_embed)
endif ()


file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/file_embed/${c_name}.c
${output_c})

file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/file_embed/${c_name}.h
${output_h})

set(${generated_c} ${CMAKE_CURRENT_BINARY_DIR}/file_embed/${c_name}.c PARENT_SCOPE)

endfunction()

if (RUN_FILE_EMBED_GENERATE)
FileEmbedGenerate(${FILE_EMBED_GENERATE_PATH} var)
endif ()
13 changes: 8 additions & 5 deletions data/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ if(NOT IS_DIRECTORY ${PROJ_DB_CACHE_DIR})
endif()
endif()

set(CONFIG_FILES
set(PROJ_INI
proj.ini
)

Expand Down Expand Up @@ -62,7 +62,7 @@ add_custom_command(
add_custom_target(generate_proj_db ALL DEPENDS ${PROJ_DB})

if(NOT "${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}")
foreach(FILE ${CONFIG_FILES} ${PROJ_DICTIONARY} ${GRIDSHIFT_FILES})
foreach(FILE ${PROJ_INI} ${PROJ_DICTIONARY} ${GRIDSHIFT_FILES})
configure_file(${FILE} ${FILE} COPYONLY)
endforeach()
endif()
Expand All @@ -80,7 +80,7 @@ set(DATA_FOR_TESTS
execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/for_tests)
execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/for_tests/tests)

foreach(FILE ${DATA_FOR_TESTS} ${CONFIG_FILES})
foreach(FILE ${DATA_FOR_TESTS} ${PROJ_INI})
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${FILE} ${CMAKE_CURRENT_BINARY_DIR}/for_tests/${FILE} COPYONLY)
endforeach()

Expand Down Expand Up @@ -111,13 +111,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/tests/conus "${CMAKE_CURRENT_BINARY_D
#install
#
set(ALL_DATA_FILE
${CONFIG_FILES}
${PROJ_DICTIONARY}
${GRIDSHIFT_FILES}
${PROJ_DB}
${SCHEMA_FILES}
${GEOTIFF_FILES}
)

if (NOT USE_ONLY_EMBEDDED_RESOURCE_FILES)
list(APPEND ALL_DATA_FILE ${PROJ_INI} ${PROJ_DB})
endif()

install(
FILES ${ALL_DATA_FILE}
DESTINATION ${PROJ_DATA_PATH}
Expand Down
25 changes: 15 additions & 10 deletions data/generate_proj_db.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ if (NOT "${PROJ_DB_SQL_MD5}" STREQUAL "${PROJ_DB_SQL_EXPECTED_MD5}")
endif()
endif()

set(generate_proj_db ON)

if(IS_DIRECTORY ${PROJ_DB_CACHE_DIR})
set(USE_PROJ_DB_CACHE_DIR TRUE)
set(PROJ_DB_SQL_MD5_FILE "${PROJ_DB_CACHE_DIR}/proj.db.sql.md5")
Expand All @@ -58,18 +60,21 @@ if(IS_DIRECTORY ${PROJ_DB_CACHE_DIR})
message(STATUS "Reusing cached proj.db from ${PROJ_DB_CACHE_DIR}")
get_filename_component(PROJ_DB_DIR "${PROJ_DB}" DIRECTORY)
file(COPY "${CACHED_PROJ_DB}" DESTINATION "${PROJ_DB_DIR}")
return()
file(TOUCH "${PROJ_DB}")
set(generate_proj_db OFF)
endif()
endif()

execute_process(COMMAND "${EXE_SQLITE3}" "${PROJ_DB}"
INPUT_FILE "${ALL_SQL_IN}"
RESULT_VARIABLE STATUS)
if (generate_proj_db)
execute_process(COMMAND "${EXE_SQLITE3}" "${PROJ_DB}"
INPUT_FILE "${ALL_SQL_IN}"
RESULT_VARIABLE STATUS)

if(STATUS AND NOT STATUS EQUAL 0)
message(FATAL_ERROR "Build of proj.db failed")
elseif(USE_PROJ_DB_CACHE_DIR)
message(STATUS "Saving cache: ${CACHED_PROJ_DB}")
file(COPY "${PROJ_DB}" DESTINATION "${PROJ_DB_CACHE_DIR}")
file(WRITE "${PROJ_DB_SQL_MD5_FILE}" "${PROJ_DB_SQL_MD5}\n")
if(STATUS AND NOT STATUS EQUAL 0)
message(FATAL_ERROR "Build of proj.db failed")
elseif(USE_PROJ_DB_CACHE_DIR)
message(STATUS "Saving cache: ${CACHED_PROJ_DB}")
file(COPY "${PROJ_DB}" DESTINATION "${PROJ_DB_CACHE_DIR}")
file(WRITE "${PROJ_DB_SQL_MD5_FILE}" "${PROJ_DB_SQL_MD5}\n")
endif()
endif()
20 changes: 20 additions & 0 deletions docs/source/install.rst
Original file line number Diff line number Diff line change
Expand Up @@ -446,6 +446,26 @@ All cached entries can be viewed using ``cmake -LAH`` from a build directory.

Embed ``PROJ_DATA`` hard-coded alternative path for data files location. Disable to avoid setting this non-relocatable hard-coded path. Default ON.

.. option:: EMBED_RESOURCE_FILES=ON/OFF

.. versionadded:: 9.6

Default is OFF for shared library builds (BUILD_SHARED_LIBS=ON), and ON
for static library builds (BUILD_SHARED_LIBS=OFF).
When ON, :file:`proj.db` and :file:`proj.ini` will be embedded into the PROJ library.

.. option:: USE_ONLY_EMBEDDED_RESOURCE_FILES=ON/OFF

.. versionadded:: 9.6

Even if EMBED_RESOURCE_FILES=ON, by default PROJ will still try to locate
:file:`proj.db` and :file:`proj.ini` on the file system, and fallback to the
embedded version if not found.
By setting USE_ONLY_EMBEDDED_RESOURCE_FILES=ON, no attempt at locating
those files on the file system is made. Default is OFF.
Users will also typically want to set EMBED_PROJ_DATA_PATH=OFF if setting
USE_ONLY_EMBEDDED_RESOURCE_FILES=OFF.


Building on Windows with vcpkg and Visual Studio 2017 or 2019
--------------------------------------------------------------------------------
Expand Down
34 changes: 34 additions & 0 deletions src/embedded_resources.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#include "embedded_resources.h"

#if USE_SHARP_EMBED
const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize) {
static const unsigned char proj_db[] = {
#embed PROJ_DB
};
*pnSize = (unsigned int)sizeof(proj_db);
return proj_db;
}

const char *pj_get_embedded_proj_ini(unsigned int *pnSize) {
static const char proj_ini[] = {
#embed PROJ_INI
};
*pnSize = (unsigned int)sizeof(proj_ini);
return proj_ini;
}

#else

#include "file_embed/proj_db.h"
const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize) {
*pnSize = proj_db_size;
return proj_db_data;
}

#include "file_embed/proj_ini.h"
const char *pj_get_embedded_proj_ini(unsigned int *pnSize) {
*pnSize = proj_ini_size;
return (const char *)proj_ini_data;
}

#endif
15 changes: 15 additions & 0 deletions src/embedded_resources.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#ifndef EMBEDDED_RESOURCES_H
#define EMBEDDED_RESOURCES_H

#ifdef __cplusplus
extern "C" {
#endif

const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize);
const char *pj_get_embedded_proj_ini(unsigned int *pnSize);

#ifdef __cplusplus
}
#endif

#endif /* EMBEDDED_RESOURCES_H */
Loading

0 comments on commit 463d104

Please sign in to comment.