From b300b3c48eb68ea715f648c2c5121e328e3d4e87 Mon Sep 17 00:00:00 2001 From: Sami Boukortt Date: Tue, 27 Aug 2024 15:28:18 +0200 Subject: [PATCH 01/31] Expose a wrapper function instead of Allotment (#3770) This lets us detect and handle failure in initialisation without requiring that we make Allotment movable for use with JXL_ASSIGN_OR_RETURN. (cherry picked from commit 878b9bd98db1c46eb319f72d83361442f37dcb8a) From 536328c0aa3ed9a0938d46287fa05acae31ca8c1 Mon Sep 17 00:00:00 2001 From: Sami Boukortt Date: Tue, 27 Aug 2024 16:19:35 +0200 Subject: [PATCH 02/31] Avoid direct use of .value_() (#3771) (cherry picked from commit 21ecfa3a4046551d893fa04bd6f238dadfb9b728) From 3ef0141e85237be6c6156ba51b3a384daca4a522 Mon Sep 17 00:00:00 2001 From: 0fbcb238c0 <24275722+0fbcb238c0@users.noreply.github.com> Date: Tue, 27 Aug 2024 16:55:52 +0200 Subject: [PATCH 03/31] Update software_support.md (#3769) Added a line describing rawtherapee's support of jxl (cherry picked from commit 308487cb81d3e6b293edbf6f85a2fc77b4b64589) From ce0ab737ffed23d284780350bee76f0b10d0d256 Mon Sep 17 00:00:00 2001 From: Leo Izen Date: Tue, 27 Aug 2024 11:12:10 -0400 Subject: [PATCH 04/31] tools/file_io.h: set binary mode for stdin and stdout on Windows (#3750) Windows stdin and stdout default to text mode, but they can be changed to binary mode with _setmode from . This is relevant because text mode enables LF <-> CRLF translation, and binary mode disables it, and it needs to be disabled for image reading/writing. This is technically also true on Unix-like operating systems, but text and binary do the same thing on macOS, Linux, etc. Signed-off-by: Leo Izen (cherry picked from commit 309b637d8a2a78708a2b806d9523ae0de562a997) --- tools/file_io.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/file_io.h b/tools/file_io.h index 266b349a..e420a7e4 100644 --- a/tools/file_io.h +++ b/tools/file_io.h @@ -20,6 +20,11 @@ #include "lib/base/compiler_specific.h" +#ifdef _WIN32 +#include +#include +#endif + namespace jpegxl { namespace tools { @@ -35,11 +40,14 @@ class FileWrapper { close_on_delete_(pathname != "-") { #ifdef _WIN32 struct __stat64 s = {}; - const int err = _stat64(pathname.c_str(), &s); + int err = _stat64(pathname.c_str(), &s); const bool is_file = (s.st_mode & S_IFREG) != 0; + if (pathname == "-") { + err |= _setmode(_fileno(file_), _O_BINARY); + } #else struct stat s = {}; - const int err = stat(pathname.c_str(), &s); + int err = stat(pathname.c_str(), &s); const bool is_file = S_ISREG(s.st_mode); #endif if (err == 0 && is_file) { From fbfe540bc2396616ee966f2e1605e478a19b078b Mon Sep 17 00:00:00 2001 From: Jon Sneyers Date: Tue, 27 Aug 2024 17:14:10 +0200 Subject: [PATCH 05/31] add way to signal Rec2100 PQ/HLG in jxl_from_tree (#3728) (cherry picked from commit 9b7ffdb03467c1bfd2a3bc9364ce3927f13261a1) From 6c4fb6ab1068d90256397b2f64a9f6105db5cd00 Mon Sep 17 00:00:00 2001 From: Jon Sneyers Date: Tue, 27 Aug 2024 17:14:37 +0200 Subject: [PATCH 06/31] add unknown and optional extra channels in PAM (#3611) (cherry picked from commit a22c0769f4002c0e683b4bb90fac05002ab80a72) --- lib/extras/dec/pnm.cc | 4 ++++ lib/extras/enc/pnm.cc | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/lib/extras/dec/pnm.cc b/lib/extras/dec/pnm.cc index 1679b2fa..9fe5d869 100644 --- a/lib/extras/dec/pnm.cc +++ b/lib/extras/dec/pnm.cc @@ -225,6 +225,10 @@ class Parser { header->ec_types.push_back(JXL_CHANNEL_CFA); } else if (MatchString("Thermal")) { header->ec_types.push_back(JXL_CHANNEL_THERMAL); + } else if (MatchString("Unknown")) { + header->ec_types.push_back(JXL_CHANNEL_UNKNOWN); + } else if (MatchString("Optional")) { + header->ec_types.push_back(JXL_CHANNEL_OPTIONAL); } else { return JXL_FAILURE("PAM: unknown TUPLTYPE"); } diff --git a/lib/extras/enc/pnm.cc b/lib/extras/enc/pnm.cc index 7e69d8e2..92a58239 100644 --- a/lib/extras/enc/pnm.cc +++ b/lib/extras/enc/pnm.cc @@ -298,6 +298,10 @@ class PAMEncoder : public BasePNMEncoder { return std::string("CFA"); case JXL_CHANNEL_THERMAL: return std::string("Thermal"); + case JXL_CHANNEL_UNKNOWN: + return std::string("Unknown"); + case JXL_CHANNEL_OPTIONAL: + return std::string("Optional"); default: return std::string("UNKNOWN"); } From 09c4e1ad0fc6328bf9af3e24f04478d210cae0a2 Mon Sep 17 00:00:00 2001 From: Moritz Firsching Date: Wed, 28 Aug 2024 12:44:05 +0200 Subject: [PATCH 07/31] adding Status to functions of PaddedBytes (#3762) * add Status to reserve * add Status to push_back * add Status to first kind of resize (part 1) * adding Status to AppendUint32 * more Status returning * add Status to the other resize * add Status to clear * add Status to assign * add Status to append and AppendKeyword * lint * don't return Status for clear * Status for TakeBytes() * Revert "Status for TakeBytes()" This reverts commit 0b750d7f53d669e6629efb77d28b1efe19bb1236. * refactor BitWriter::Allotment Co-authored-by: Sami Boukortt * No more need to set capacity and size to zero to notice a failure to reallocate * No more need for these checks either * return directly * not QUIET --------- Co-authored-by: Sami Boukortt (cherry picked from commit 637806b163fec69869a13465cb7b126cec68ae97) From fc4d21c26184efd0d4b4308ba0ffeb30cd1e7d77 Mon Sep 17 00:00:00 2001 From: Sami Boukortt Date: Thu, 29 Aug 2024 14:04:26 +0200 Subject: [PATCH 08/31] Fix/homogenise the disabling of spline tests (#3774) JXL_CRASH_ON_ERROR is always defined (which is why it was possible to use it in the runtime condition), so DuplicatePoints ended up always disabled. (cherry picked from commit 17db92d4ffb25f6963cccb30695eb3e2f7c01ef2) From e3869fbd5e6d067ef1ba9b43ec103a80d1620add Mon Sep 17 00:00:00 2001 From: Sami Boukortt Date: Thu, 29 Aug 2024 14:38:24 +0200 Subject: [PATCH 09/31] =?UTF-8?q?=20If=20InitializePPF=20fails=20in=20cjxl?= =?UTF-8?q?,=20return=20EXIT=5FFAILURE=20rather=20than=20=E2=80=9Cfalse?= =?UTF-8?q?=E2=80=9D=20(0)=20(#3780)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * If InitializePPF fails in cjxl, return EXIT_FAILURE rather than “false” (0) * Flushing std::cerr is redundant (the unitbuf flag is on by default) (cherry picked from commit 35e7376a4b895e0d9468e35cd9eb2b6eb71bf394) --- tools/fuzzer_stub.cc | 2 +- tools/jpegli_dec_fuzzer_corpus.cc | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tools/fuzzer_stub.cc b/tools/fuzzer_stub.cc index ca3977ae..3ba8cbb8 100644 --- a/tools/fuzzer_stub.cc +++ b/tools/fuzzer_stub.cc @@ -19,7 +19,7 @@ void ProcessInput(const char* filename) { std::vector contents((std::istreambuf_iterator(ifs)), std::istreambuf_iterator()); ifs.close(); - std::cout << "Processing " << filename << "\n" << std::flush; + std::cout << "Processing " << filename << "\n"; LLVMFuzzerTestOneInput(reinterpret_cast(contents.data()), contents.size()); } diff --git a/tools/jpegli_dec_fuzzer_corpus.cc b/tools/jpegli_dec_fuzzer_corpus.cc index ab291ddf..88e499a4 100644 --- a/tools/jpegli_dec_fuzzer_corpus.cc +++ b/tools/jpegli_dec_fuzzer_corpus.cc @@ -220,8 +220,7 @@ bool GenerateFile(const char* output_dir, const ImageSpec& spec, if (!quiet) { std::unique_lock lock(stderr_mutex); - std::cerr << "Generating " << spec << " as " << hash_str << "\n" - << std::flush; + std::cerr << "Generating " << spec << " as " << hash_str << "\n"; } uint8_t hash[16]; @@ -248,8 +247,7 @@ bool GenerateFile(const char* output_dir, const ImageSpec& spec, if (!quiet) { std::unique_lock lock(stderr_mutex); std::cerr << "Stored " << output_fn << " size: " << compressed.size() - << "\n" - << std::flush; + << "\n"; } return true; @@ -342,7 +340,7 @@ int main(int argc, const char** argv) { spec.seed = mt() % 777777; if (!spec.Validate()) { if (!quiet) { - std::cerr << "Skipping " << spec << "\n" << std::flush; + std::cerr << "Skipping " << spec << "\n"; } } else { specs.push_back(spec); @@ -365,9 +363,9 @@ int main(int argc, const char** argv) { }; if (!RunOnPool(pool.get(), 0, specs.size(), jxl::ThreadPool::NoInit, generate, "FuzzerCorpus")) { - std::cerr << "Error generating fuzzer corpus\n" << std::flush; + std::cerr << "Error generating fuzzer corpus\n"; return EXIT_FAILURE; } - std::cerr << "Finished generating fuzzer corpus\n" << std::flush; + std::cerr << "Finished generating fuzzer corpus\n"; return EXIT_SUCCESS; } From 19928a2c7829a2f75267094d2055093d75e23b3f Mon Sep 17 00:00:00 2001 From: Moritz Firsching Date: Thu, 29 Aug 2024 15:36:53 +0200 Subject: [PATCH 10/31] stop loading uninitialized memory from MaskingPtr1x1 (#3778) * stop loading uninitialized memory from MaskingPtr1x1 * use mask1x1_xsize (cherry picked from commit 0d303cc6c3bb198c76c0ae182c515cf35d02f97e) From 8f9ce8a565e96903d586ed86e07ed82a3a23c67f Mon Sep 17 00:00:00 2001 From: monad0 Date: Thu, 29 Aug 2024 07:14:54 -0700 Subject: [PATCH 11/31] Reduce compute and increase density of e11 (#3779) * Reduce compute and increase density of e11 First, run a test to differentiate images by palette performance (e.g. photo vs text), then run trials optimized for that content. 84 images, 20 threads: ``` bpp (mean) unique mins Mpx/s real (mean) B | mins | Mpx/s CPU (mean) best of 7320243 3.488567 65.48% 40.48% 0.004631646 0.02237144 cjxl_0.11.0-637806b1-p24t21_d0e11 7325717 3.488910 59.52% 34.52% 0.00025005346 0.004543083 cjxl_0.11.0-637806b1_d0e11 ``` * Update AUTHORS * Provide functions for content settings in e11 (cherry picked from commit 95246e5387d8c9ee2554ac37627ea4172a143657) --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index a496f4e9..6091defa 100644 --- a/AUTHORS +++ b/AUTHORS @@ -39,6 +39,7 @@ Aryan Pingle Biswapriyo Nath CanadianBaconBoi Damiano Albani +Damon Townsend Daniel Novomeský David Burnett dependabot[bot] From 99d2625a19329fb42ad101ed6e166a9cff1917d6 Mon Sep 17 00:00:00 2001 From: Sami Boukortt Date: Thu, 29 Aug 2024 16:45:51 +0200 Subject: [PATCH 12/31] Improve error messages (#3781) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Message when InitializePPF fails * Don’t add unnecessary -- to error messages related to flag parsing Before: $ tools/cjxl --streaming_input=0 in.ppm out.jxl ----streaming_input=0 didn't expect any argument passed to it. Error parsing flag --streaming_input=0 Use 'tools/cjxl -h' for more information $ tools/cjxl in.ppm out.jxl -d ---d expected an argument but none passed. Error parsing flag -d Use 'tools/cjxl -h' for more information After: $ tools/cjxl --streaming_input=0 in.ppm out.jxl --streaming_input didn't expect any argument passed to it. Error parsing flag --streaming_input=0 Use 'tools/cjxl -h' for more information $ tools/cjxl in.ppm out.jxl -d -d expected an argument but none passed. Error parsing flag -d Use 'tools/cjxl -h' for more information (cherry picked from commit 84f862853221fa09647279f85814f111e1f32fae) --- tools/cmdline.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/cmdline.h b/tools/cmdline.h index b202fc4a..fa7c92f3 100644 --- a/tools/cmdline.h +++ b/tools/cmdline.h @@ -271,7 +271,7 @@ class CommandLineParser { return (*parser_.parser_with_arg_)(arg, storage_); } else { fprintf(stderr, "--%s didn't expect any argument passed to it.\n", - argv[*i]); + long_name_); return false; } } @@ -280,7 +280,7 @@ class CommandLineParser { (*i)++; if (metavar_) { if (argc <= *i) { - fprintf(stderr, "--%s expected an argument but none passed.\n", + fprintf(stderr, "%s expected an argument but none passed.\n", argv[*i - 1]); return false; } From 03f167943b9adde752e9b47f84bff401ee023065 Mon Sep 17 00:00:00 2001 From: monad0 Date: Thu, 29 Aug 2024 09:09:43 -0700 Subject: [PATCH 13/31] Fix e11 iterations (#3782) 84 images, 20 threads: ``` bpp (mean) unique mins Mpx/s real (mean) B | mins | Mpx/s CPU (mean) best of 7318553 3.482824 73.81% 42.86% 0.003986642 0.02157361 cjxl_0.11.0-637806b1-p24t21-fix_d0e11 7325717 3.488910 57.14% 26.19% 0.00025005346 0.004543083 cjxl_0.11.0-637806b1_d0e11 ``` (cherry picked from commit 579837b5d406be94a300d37be965e6f0534f3390) From 177a428fa84fdc21aa87eaa77c738691c0a92336 Mon Sep 17 00:00:00 2001 From: Moritz Firsching Date: Fri, 30 Aug 2024 09:44:50 +0200 Subject: [PATCH 14/31] ClangTidy fixes (#3783) * ClangTidy fixes * format (cherry picked from commit 65f06314aa78751312cc7c8fa7a4d260a4c2b2c3) --- lib/extras/memory_manager_internal.h | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/extras/memory_manager_internal.h b/lib/extras/memory_manager_internal.h index 9cc2d407..bb8d72b8 100644 --- a/lib/extras/memory_manager_internal.h +++ b/lib/extras/memory_manager_internal.h @@ -10,6 +10,7 @@ #include #include +#include #include "lib/base/compiler_specific.h" #include "lib/base/memory_manager.h" From dbe8fb0a23c75380665ee26b6f4c05e925cbb87c Mon Sep 17 00:00:00 2001 From: Moritz Firsching Date: Fri, 30 Aug 2024 10:51:40 +0200 Subject: [PATCH 15/31] more ClangTidy fixes (#3784) (cherry picked from commit 9ffc2e0fc1a31513db01eb1c458310ad71a475f8) From bdbabafd28941de33e19698181e5b8169364efec Mon Sep 17 00:00:00 2001 From: Moritz Firsching Date: Fri, 30 Aug 2024 13:06:30 +0200 Subject: [PATCH 16/31] even more ClangTidy fixes (#3785) (cherry picked from commit db689b27648bf5a657dee0e2065aad5f3d36f2c8) --- tools/tracking_memory_manager.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/tracking_memory_manager.cc b/tools/tracking_memory_manager.cc index 704ec0e6..5d0ac64a 100644 --- a/tools/tracking_memory_manager.cc +++ b/tools/tracking_memory_manager.cc @@ -5,6 +5,8 @@ #include "tools/tracking_memory_manager.h" +#include +#include #include #include From 4efb89f2a9d68d383044f02cd70e9911c9a168f0 Mon Sep 17 00:00:00 2001 From: Jyrki Alakuijala Date: Fri, 30 Aug 2024 17:18:02 +0200 Subject: [PATCH 17/31] encoding changes for more precise epf (#3786) about 0.15 % on BPP*pnorm ``` Before: 104 images Encoding kPixels Bytes BPP E MP/s D MP/s Max norm SSIMULACRA2 PSNR pnorm BPP*pnorm QABPP Bugs ---------------------------------------------------------------------------------------------------------------------------------------- jxl:d0.1 20315 17234055 6.7864714 0.795 8.565 0.26921220 95.72922587 56.47 0.09149990 0.620961464563 6.786 0 jxl:d0.5 20315 7112515 2.8007848 0.940 12.078 0.85998310 91.46395356 46.44 0.34487223 0.965912919133 2.941 0 jxl:d0.8 20315 5380024 2.1185600 0.861 11.031 1.10730124 88.87286260 44.43 0.46413467 0.983297134681 2.471 0 jxl:d1 20315 4616590 1.8179329 0.861 11.503 1.32266768 87.04284922 43.20 0.55278192 1.004920452626 2.499 0 jxl:d1.5 20315 3501542 1.3788464 0.879 11.142 1.84924045 82.85156040 41.07 0.75456129 1.040424113923 2.644 0 jxl:d2.0 20315 2846638 1.1209566 0.881 11.101 2.40151432 78.80988790 39.56 0.94325779 1.057351028760 2.835 0 jxl:d2.5 20315 2396802 0.9438190 0.893 11.413 2.82912128 74.97206547 38.37 1.11707140 1.054313198037 2.814 0 jxl:d3 20315 2085594 0.8212707 0.870 11.541 3.25567966 71.58518909 37.51 1.27852510 1.050015177507 2.801 0 jxl:d5 20315 1398149 0.5505668 0.873 7.995 4.58800935 60.07606195 35.23 1.80660922 0.994659027562 2.622 0 jxl:d10 20315 843981 0.3323451 0.865 8.538 7.68431602 38.42064210 32.46 2.92598859 0.972437834700 2.534 0 Aggregate: 20315 3351858 1.3199034 0.871 10.385 1.85750610 76.98242982 41.01 0.73089142 0.964706041010 2.941 0 After: 104 images Encoding kPixels Bytes BPP E MP/s D MP/s Max norm SSIMULACRA2 PSNR pnorm BPP*pnorm QABPP Bugs ---------------------------------------------------------------------------------------------------------------------------------------- jxl:d0.1 20315 17266997 6.7994434 0.815 8.802 0.26883625 95.74016251 56.48 0.09126255 0.620534568214 6.799 0 jxl:d0.5 20315 7125011 2.8057055 0.962 12.895 0.85243631 91.45559592 46.45 0.34403907 0.965272309787 2.931 0 jxl:d0.8 20315 5394265 2.1241678 0.894 11.084 1.09960077 88.89451775 44.49 0.46027287 0.977696821726 2.466 0 jxl:d1 20315 4624207 1.8209324 0.892 11.385 1.31717842 87.04791994 43.25 0.54900805 0.999706533924 2.495 0 jxl:d1.5 20315 3504409 1.3799754 0.912 11.012 1.85527307 82.85677175 41.09 0.75216350 1.037967114212 2.655 0 jxl:d2.0 20315 2844789 1.1202285 0.923 11.199 2.40613996 78.78007662 39.57 0.94250548 1.055821484127 2.840 0 jxl:d2.5 20315 2393011 0.9423262 0.923 11.503 2.83238352 74.92339850 38.38 1.11739846 1.052953789398 2.809 0 jxl:d3 20315 2079292 0.8187891 0.891 11.957 3.25359934 71.51771710 37.50 1.28061805 1.048556056115 2.794 0 jxl:d5 20315 1395949 0.5497005 0.903 8.354 4.62224705 60.04091153 35.21 1.81152423 0.995795707602 2.640 0 jxl:d10 20315 842166 0.3316303 0.892 8.832 7.70488447 38.44032069 32.44 2.93156275 0.972195146095 2.540 0 Aggregate: 20315 3351787 1.3198755 0.900 10.601 1.85647607 76.96973923 41.02 0.72960225 0.962984133078 2.942 0 ``` (cherry picked from commit f28befab0b0ef113ffdca3786757737bce01d4c7) --- tools/optimizer/simplex_fork.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/tools/optimizer/simplex_fork.py b/tools/optimizer/simplex_fork.py index ff0bc78e..310357ba 100755 --- a/tools/optimizer/simplex_fork.py +++ b/tools/optimizer/simplex_fork.py @@ -60,12 +60,16 @@ def EvalCacheForget(): global eval_hash eval_hash = {} +g_first_pnorm = None + def RandomizedJxlCodecs(): + global g_first_pnorm + g_first_pnorm = None retval = [] - minval = 0.2 - maxval = 2.3 + minval = 1.0 + maxval = 8.3 rangeval = maxval/minval - steps = 3 + steps = 5 for i in range(steps): mul = minval * rangeval**(float(i)/(steps - 1)) mul *= 0.99 + 0.05 * random.random() @@ -89,6 +93,7 @@ def Eval(vec, binary_name, cached=True): global eval_hash global g_codecs global g_best_val + global g_first_pnorm key = "" # os.environ["BUTTERAUGLI_OPTIMIZE"] = "1" for i in range(300): @@ -113,7 +118,7 @@ def Eval(vec, binary_name, cached=True): # process.wait() found_score = False - vec[0] = 1.0 + vec[0] = 1e29 dct2 = 0.0 dct4 = 0.0 dct16 = 0.0 @@ -122,17 +127,14 @@ def Eval(vec, binary_name, cached=True): for line in process.communicate(input=None)[0].splitlines(): print("BE", line) sys.stdout.flush() - if line[0:3] == b'jxl': - bpp = line.split()[3] - dist_pnorm = line.split()[9] - dist_max = line.split()[6] - vec[0] *= float(dist_pnorm) * float(bpp) / 16.0 - #vec[0] *= (float(dist_max) * float(bpp) / 16.0) ** 0.01 - n += 1 + if line[0:4] == b'Aggr': + bpppnorm = float(line.split()[10]) + if g_first_pnorm == None: + g_first_pnorm = float(line.split()[9]) + correctbpp = 1. + 10.0 * (float(line.split()[9]) - g_first_pnorm) ** 2 + + vec[0] = 1e-5 * bpppnorm * correctbpp found_score = True - distance = float(line.split()[0].split(b'd')[-1]) - faultybpp = 1.0 + 0.43 * ((float(bpp) * distance ** 0.69) - 1.64) ** 2 - vec[0] *= faultybpp print("eval: ", vec) if (vec[0] <= 0.0): @@ -257,6 +259,7 @@ def InitialSimplex(vec, dim, amount): mulli = 0.1 + 15 * random.random()**2.0 g_codecs = RandomizedJxlCodecs() + g_best_val = None # get a new best val print("\n\n\nRestart", restarts, "mulli", mulli) g_simplex.sort() best = g_simplex[0][:] From 4dc04dab102356399d2ade3b19cb5014a0e552e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Sep 2024 08:09:35 +0200 Subject: [PATCH 18/31] Bump github/codeql-action from 3.26.2 to 3.26.6 (#3790) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.26.2 to 3.26.6. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/429e1977040da7a23b6822b13c129cd1ba93dbb2...4dd16135b69a43b6c8efb853346f8437d92d3c93) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit 2650e5cbc0df13735b9ed96f066cc3ff315c8c7b) From 28d9fc864d47e132887f29d830916375c1226c62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Sep 2024 06:35:50 +0000 Subject: [PATCH 19/31] Bump actions/upload-artifact from 4.3.6 to 4.4.0 (#3791) Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4.3.6 to 4.4.0. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/834a144ee995460fba8ed112a2fc961b36a5ec5a...50769540e7f4bd5e21e526ee35c689e35e0d6874) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit 960d97c0b73fe081a7cd00afc3fac1bfaf441452) --- .github/workflows/build_test_cross.yml | 4 ++-- .github/workflows/fuzz.yml | 2 +- .github/workflows/release.yaml | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_test_cross.yml b/.github/workflows/build_test_cross.yml index 578a6716..6328b265 100644 --- a/.github/workflows/build_test_cross.yml +++ b/.github/workflows/build_test_cross.yml @@ -184,7 +184,7 @@ jobs: mkdir -p ./build/Testing/Temporary unzip ./tools/scripts/test_cost-${{ matrix.identifier }}.zip -d ./build/Testing/Temporary - - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 + - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 if: env.WILL_RUN_TESTS == 'true' with: name: cross_binary-${{ matrix.identifier }} @@ -295,7 +295,7 @@ jobs: -I ${{ matrix.shard_number }},,${{ env.LAST_SHARD }} \ -E '(bash_test|conformance_tooling_test|test_jxl_jni_wrapper)' - - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 + - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 name: Upload test cost if: (env.UPLOAD_TEST_COST == 'true') && (matrix.shard_number == 0) && (env.WILL_RUN_TESTS == 'true') with: diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 3d1e04aa..52f296de 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -61,7 +61,7 @@ jobs: language: c++ fuzz-seconds: 600 - name: Upload Crash - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 if: failure() && steps.build.outcome == 'success' with: name: artifacts diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7717000f..3bf4d846 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -100,7 +100,7 @@ jobs: ${{ runner.workspace }}/jpegli-linux-x86_64-static-${{ github.event.release.tag_name }}.tar.gz - name: Upload artifacts - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: name: jpegli-linux-x86_64-static path: ${{ runner.workspace }}/release_file.tar.gz @@ -216,7 +216,7 @@ jobs: ./ci.sh debian_stats - name: Upload artifacts - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: name: ${{ steps.env.outputs.artifact_name }} path: | @@ -364,7 +364,7 @@ jobs: BUILD_CONFIG=Release/ BENCHMARK_NUM_THREADS=2 STORE_IMAGES=0 ./ci.sh fast_benchmark - name: Upload artifacts - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: name: jpegli-${{matrix.triplet}} path: | From 19f9d72297ed2d2b71ec86faa621f5e772a13c8c Mon Sep 17 00:00:00 2001 From: Moritz Firsching Date: Thu, 5 Sep 2024 10:01:14 +0200 Subject: [PATCH 20/31] exclude test_jpegli_jni_wrapper (#3797) (cherry picked from commit 43221c84373494a21cab532e7bc2f51128c73207) --- .github/workflows/build_test_cross.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_cross.yml b/.github/workflows/build_test_cross.yml index 6328b265..4f15a2f1 100644 --- a/.github/workflows/build_test_cross.yml +++ b/.github/workflows/build_test_cross.yml @@ -293,7 +293,7 @@ jobs: chmod +x `cat executable.lst` ./ci.sh test \ -I ${{ matrix.shard_number }},,${{ env.LAST_SHARD }} \ - -E '(bash_test|conformance_tooling_test|test_jxl_jni_wrapper)' + -E '(bash_test|conformance_tooling_test|test_jxl_jni_wrapper|test_jpegli_jni_wrapper)' - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 name: Upload test cost From d9d9dab5c3f1998271ea48052308316427fbb028 Mon Sep 17 00:00:00 2001 From: Moritz Firsching Date: Thu, 5 Sep 2024 12:40:03 +0200 Subject: [PATCH 21/31] fix tests from #3786 (#3796) (cherry picked from commit 724646620d1d2e706c30876fa94ba94f14f6dda3) From 4fde3b7975dcd1fbbf4a09e60804c75d5abc0692 Mon Sep 17 00:00:00 2001 From: Sami Boukortt Date: Thu, 5 Sep 2024 14:18:18 +0200 Subject: [PATCH 22/31] In static builds, also link the MSVC runtime statically (#3803) This might help with #3666. (cherry picked from commit b5cb2464c6d42c3688b4f51e5101adf2ae291bfd) --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c71c904..0dcabd05 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -190,6 +190,11 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) if(JPEGXL_STATIC) set(BUILD_SHARED_LIBS 0) + + # https://learn.microsoft.com/en-us/cpp/build/reference/md-mt-ld-use-run-time-library?view=msvc-170 + # https://cmake.org/cmake/help/latest/variable/CMAKE_MSVC_RUNTIME_LIBRARY.html + set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded) + # Clang developers say that in case to use "static" we have to build stdlib # ourselves; for real use case we don't care about stdlib, as it is "granted", # so just linking all other libraries is fine. From 23f76ba759ab2666148897d491c11e1ec3294de5 Mon Sep 17 00:00:00 2001 From: Moritz Firsching Date: Thu, 5 Sep 2024 14:40:36 +0200 Subject: [PATCH 23/31] check realloc in basicinfo (#3798) * check realloc in basicinfo * format * reset box_foo * no need for else (cherry picked from commit 4d77514fc8ce156cb135238248502ca3eeab8b2e) From 11f585ae82d8c099a5130f3c66cb8d85d4e0a2fc Mon Sep 17 00:00:00 2001 From: Moritz Firsching Date: Thu, 5 Sep 2024 16:22:59 +0200 Subject: [PATCH 24/31] disable test_jpegli_jni_wrapper in converage test (#3802) (cherry picked from commit 3786abba487582b22f484bfb22832d31af62bc84) --- .github/workflows/build_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 45105cc0..ccc7d740 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -77,6 +77,8 @@ jobs: - name: coverage env_test_stack_size: 2048 skip_install: true + # TODO: understand why this does not work + ctest_args: -E 'test_jpegli_jni_wrapper' # Build with support for decoding to JPEG bytes disabled. Produces a # smaller build if only decoding to pixels is needed. - name: release-nojpeg From 679f0e4eb5a5667be98080128df18fa77590aa5a Mon Sep 17 00:00:00 2001 From: Leo Izen Date: Mon, 9 Sep 2024 07:37:59 -0400 Subject: [PATCH 25/31] tools/benchmark_codec_avif.cc: restore compatability with libavif < 1 (#3805) Fixes regression since 5df155d975bd8ecf091f2b808d41cdd1f37e09b8. This commit restores compatibility with these functions using libavif before major version 1, as they didn't return a result in those cases. Signed-off-by: Leo Izen (cherry picked from commit 4a24ceb55dacba6b0308f14ffdee206a11f47720) From d47998d430de59d51d7a9d45f48775e124017556 Mon Sep 17 00:00:00 2001 From: Sami Boukortt Date: Tue, 10 Sep 2024 15:03:14 +0200 Subject: [PATCH 26/31] Add a couple of missing HWY_ATTR (#3810) This might address #3807. (cherry picked from commit 95123121c1e3a0f3a8961d153480aae9864cbf99) --- lib/extras/convolve-inl.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/extras/convolve-inl.h b/lib/extras/convolve-inl.h index 67a90e4c..76aa0175 100644 --- a/lib/extras/convolve-inl.h +++ b/lib/extras/convolve-inl.h @@ -243,8 +243,8 @@ class ConvolveT { const Weights& weights, ThreadPool* pool, ImageF* out) { const int64_t stride = in.PixelsPerRow(); - const auto process_row = [&](const uint32_t y, - size_t /*thread*/) -> Status { + const auto process_row = [&](const uint32_t y, size_t /*thread*/) + HWY_ATTR -> Status { RunRow(rect.ConstRow(in, y), rect.xsize(), stride, WrapRowUnchanged(), weights, out->Row(y)); return true; @@ -263,8 +263,8 @@ class ConvolveT { const Weights& weights, ThreadPool* pool, Image3F* out) { const int64_t stride = in.PixelsPerRow(); - const auto process_row = [&](const uint32_t y, - size_t /*thread*/) -> Status { + const auto process_row = [&](const uint32_t y, size_t /*thread*/) + HWY_ATTR -> Status { for (size_t c = 0; c < 3; ++c) { RunRow(rect.ConstPlaneRow(in, c, y), rect.xsize(), stride, WrapRowUnchanged(), weights, out->PlaneRow(c, y)); From 76f9903984cae0715510f434eb170ded6286a582 Mon Sep 17 00:00:00 2001 From: Sami Boukortt Date: Tue, 10 Sep 2024 18:35:52 +0200 Subject: [PATCH 27/31] Fix build with old gcc (#3811) (cherry picked from commit a4f67292d1cd99f969ad86aa5ceb272a6bec737a) --- lib/extras/convolve-inl.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/extras/convolve-inl.h b/lib/extras/convolve-inl.h index 76aa0175..31f4aed6 100644 --- a/lib/extras/convolve-inl.h +++ b/lib/extras/convolve-inl.h @@ -243,8 +243,7 @@ class ConvolveT { const Weights& weights, ThreadPool* pool, ImageF* out) { const int64_t stride = in.PixelsPerRow(); - const auto process_row = [&](const uint32_t y, size_t /*thread*/) - HWY_ATTR -> Status { + const auto process_row = [&](const uint32_t y, size_t /*thread*/) HWY_ATTR { RunRow(rect.ConstRow(in, y), rect.xsize(), stride, WrapRowUnchanged(), weights, out->Row(y)); return true; @@ -263,8 +262,7 @@ class ConvolveT { const Weights& weights, ThreadPool* pool, Image3F* out) { const int64_t stride = in.PixelsPerRow(); - const auto process_row = [&](const uint32_t y, size_t /*thread*/) - HWY_ATTR -> Status { + const auto process_row = [&](const uint32_t y, size_t /*thread*/) HWY_ATTR { for (size_t c = 0; c < 3; ++c) { RunRow(rect.ConstPlaneRow(in, c, y), rect.xsize(), stride, WrapRowUnchanged(), weights, out->PlaneRow(c, y)); From 2c78a810f33df8f2493a07a6498f3202dcdb279f Mon Sep 17 00:00:00 2001 From: Jonathan Brown Date: Fri, 13 Sep 2024 12:29:21 +0100 Subject: [PATCH 28/31] Faster PNG compression (#3819) * Lower PNG compression * Try level 1 (cherry picked from commit 0185fcd0df1b3901972752172b188e3c085777f8) --- third_party/apngdis/enc.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/third_party/apngdis/enc.cc b/third_party/apngdis/enc.cc index c7b18961..688ef1c6 100644 --- a/third_party/apngdis/enc.cc +++ b/third_party/apngdis/enc.cc @@ -381,6 +381,7 @@ Status APNGEncoder::EncodePackedPixelFileToAPNG( info_ptr = png_create_info_struct(png_ptr); if (!info_ptr) return JXL_FAILURE("Could not init png info struct"); + png_set_compression_level(png_ptr, 1); png_set_write_fn(png_ptr, bytes, PngWrite, nullptr); png_set_flush(png_ptr, 0); From 7e0938006cfa50752fc9bba7206c11de204dd6e0 Mon Sep 17 00:00:00 2001 From: Moritz Firsching Date: Fri, 13 Sep 2024 13:56:25 +0200 Subject: [PATCH 29/31] clarify release instructions (#3820) (cherry picked from commit 33f7c03d89a33e24d7504acfa485cf246252e8bb) --- doc/release.md | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/doc/release.md b/doc/release.md index e1dcb9d8..3448640f 100644 --- a/doc/release.md +++ b/doc/release.md @@ -93,6 +93,15 @@ branches](https://docs.github.com/en/github/administering-a-repository/defining- in our repository, however you can push to a protected branch when *creating* it but you can't directly push to it after it is created. To include more changes in the release branch see the "Cherry-picking fixes to a release" section below. +Remember to tag the first commit in the `main` branch after the branch of point +with the suffix `-snapshot`, so for instances, if you just made the +branch `v0.5.x`, run +``` +git checkout [hash of first commit after the branch off point] +git tag v0.5-snapshot +git push git@github.com:libjxl/libjxl.git v0.5-snapshot +``` + ## Creating a merge label @@ -274,17 +283,29 @@ instructions: the release itself that are not included in the CHANGELOG.md, although prefer to include those in the CHANGELOG.md file. You can switch to the Preview tab to see the results. + * Copy-paste the following note: + ```markdown + **Note:** This release is for evaluation purposes and may contain bugs, including security bugs, that may not be individually documented when fixed. See the [SECURITY.md](SECURITY.md) file for details. Always prefer to use the latest release. + +Please provide feedback and report bugs [here](https://github.com/libjxl/libjxl/issues). +``` + * Finally click "Publish release" and go celebrate with the team. 🎉 * The branch v0.7.x will be pushed to gitlab automatically, but make sure to manually push the *tag* of the release also to - https://gitlab.com/wg1/jpeg-xl, by doing + https://gitlab.com/wg1/jpeg-xl, by doing configuring `gitlab` to be the remote `git@gitlab.com:wg1/jpeg-xl.git` and then running: ```bash git push gitlab v0.7.1 ``` -where `gitlab` is the remote `git@gitlab.com:wg1/jpeg-xl.git`. +and possibly also push the tag for the snapshot to gitlab, if it wasn't done yet: +```bash +git push gitlab v0.7-snapshot +``` + + ### How to build downstream projects From 96652819debb7821d36a3371bcb428bd83adf280 Mon Sep 17 00:00:00 2001 From: Moritz Firsching Date: Fri, 13 Sep 2024 19:55:39 +0200 Subject: [PATCH 30/31] re-activate streaming fuzzer for higher effors (#3822) (cherry picked from commit 94819f129c9531fb59b9c70121ab2ceb94368ad4) From 81e958a9538faf615ff5e4d4a60b89a1a6050aad Mon Sep 17 00:00:00 2001 From: Moritz Firsching Date: Fri, 13 Sep 2024 20:19:11 +0200 Subject: [PATCH 31/31] bump version to 0.12.0 (#3817) * bump version to 0.12.0 * update JPEGXL_SO_MINOR_VERSION (cherry picked from commit 7609879f3d9071506dea8f537ad56d744ee83617) --- debian/changelog | 10 ++++++++-- lib/CMakeLists.txt | 4 ++-- lib/jxl_lists.bzl | 2 +- lib/lib.gni | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/debian/changelog b/debian/changelog index 0a58b1db..65017718 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,14 @@ -jpeg-xl (0.11.0) UNRELEASED; urgency=medium +jpeg-xl (0.12.0) UNRELEASED; urgency=medium + + * Bump JPEG XL version to 0.12.0. + + -- JPEG XL Maintainers Fri, 13 Sep 2024 09:19:23 +0200 + +jpeg-xl (0.11.0) unstable; urgency=medium * Bump JPEG XL version to 0.11.0. - -- JPEG XL Maintainers Tue, 06 Aug 2024 14:35:34 +0200 + -- JPEG XL Maintainers Fri, 13 Sep 2024 09:19:23 +0200 jpeg-xl (0.10.2) unstable; urgency=medium diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index e03093ae..df7cdb45 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -5,7 +5,7 @@ # https://developers.google.com/open-source/licenses/bsd set(JPEGXL_MAJOR_VERSION 0) -set(JPEGXL_MINOR_VERSION 11) +set(JPEGXL_MINOR_VERSION 12) set(JPEGXL_PATCH_VERSION 0) set(JPEGXL_LIBRARY_VERSION "${JPEGXL_MAJOR_VERSION}.${JPEGXL_MINOR_VERSION}.${JPEGXL_PATCH_VERSION}") @@ -16,7 +16,7 @@ set(JPEGXL_LIBRARY_VERSION # It is important to update this value when making incompatible API/ABI changes # so that programs that depend on libjxl can update their dependencies. Semantic # versioning allows 0.y.z to have incompatible changes in minor versions. -set(JPEGXL_SO_MINOR_VERSION 11) +set(JPEGXL_SO_MINOR_VERSION 12) if (JPEGXL_MAJOR_VERSION EQUAL 0) set(JPEGXL_LIBRARY_SOVERSION "${JPEGXL_MAJOR_VERSION}.${JPEGXL_SO_MINOR_VERSION}") diff --git a/lib/jxl_lists.bzl b/lib/jxl_lists.bzl index fa4a5ad3..5ca8a9c7 100644 --- a/lib/jxl_lists.bzl +++ b/lib/jxl_lists.bzl @@ -545,7 +545,7 @@ libjxl_jpegli_wrapper_sources = [ libjxl_major_version = 0 -libjxl_minor_version = 11 +libjxl_minor_version = 12 libjxl_patch_version = 0 diff --git a/lib/lib.gni b/lib/lib.gni index 44581eb3..56986285 100644 --- a/lib/lib.gni +++ b/lib/lib.gni @@ -538,7 +538,7 @@ libjxl_jpegli_wrapper_sources = [ libjxl_major_version = 0 -libjxl_minor_version = 11 +libjxl_minor_version = 12 libjxl_patch_version = 0