From 9c50f53065917249a47693041e8ab52ae96e0d06 Mon Sep 17 00:00:00 2001 From: Ruud van Asseldonk Date: Tue, 8 May 2018 13:19:14 +0200 Subject: [PATCH] squashfsTools: apply reproducibility patches Without these patches, the output of mksquashfs is not reproducible. The patches are taken from https://github.com/squashfskit/squashfskit, a fork of squashfs-tools, licensed under the GPL2 or later like squashfs-tools itself. --- ...POCH-is-set-override-timestamps-with.patch | 90 +++++++ ...POCH-is-set-also-clamp-content-times.patch | 83 +++++++ .../0003-remove-frag-deflator-thread.patch | 220 ++++++++++++++++++ pkgs/tools/filesystems/squashfs/default.nix | 9 + 4 files changed, 402 insertions(+) create mode 100644 pkgs/tools/filesystems/squashfs/0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patch create mode 100644 pkgs/tools/filesystems/squashfs/0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patch create mode 100644 pkgs/tools/filesystems/squashfs/0003-remove-frag-deflator-thread.patch diff --git a/pkgs/tools/filesystems/squashfs/0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patch b/pkgs/tools/filesystems/squashfs/0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patch new file mode 100644 index 000000000000..5626800e723c --- /dev/null +++ b/pkgs/tools/filesystems/squashfs/0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patch @@ -0,0 +1,90 @@ +From 0ab12a8585373be2de5129e14d979c62e7a90d82 Mon Sep 17 00:00:00 2001 +From: Chris Lamb +Date: Mon, 21 Nov 2016 09:33:05 +0100 +Subject: [PATCH] If SOURCE_DATE_EPOCH is set, override timestamps with that + value. + +See https://reproducible-builds.org/specs/source-date-epoch/ for more +information about this environment variable. + +Based on a patch by Alexander Couzens posted on +https://sourceforge.net/p/squashfs/mailman/message/34673610/ + +Signed-off-by: Chris Lamb +--- + squashfs-tools/mksquashfs.c | 38 ++++++++++++++++++++++++++++++++++++- + 1 file changed, 37 insertions(+), 1 deletion(-) + +diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c +index c2098bd..b49e956 100644 +--- a/squashfs-tools/mksquashfs.c ++++ b/squashfs-tools/mksquashfs.c +@@ -137,6 +137,9 @@ unsigned int cache_bytes = 0, cache_size = 0, inode_count = 0; + /* inode lookup table */ + squashfs_inode *inode_lookup_table = NULL; + ++/* override filesystem creation time */ ++time_t mkfs_fixed_time = -1; ++ + /* in memory directory data */ + #define I_COUNT_SIZE 128 + #define DIR_ENTRIES 32 +@@ -5104,6 +5107,9 @@ int main(int argc, char *argv[]) + int total_mem = get_default_phys_mem(); + int progress = TRUE; + int force_progress = FALSE; ++ char *source_date_epoch, *endptr; ++ unsigned long long epoch; ++ + struct file_buffer **fragment = NULL; + + if(argc > 1 && strcmp(argv[1], "-version") == 0) { +@@ -5641,6 +5647,36 @@ printOptions: + } + } + ++ /* if SOURCE_DATE_EPOCH is set, use that timestamp for the mkfs time */ ++ source_date_epoch = getenv("SOURCE_DATE_EPOCH"); ++ if(source_date_epoch) { ++ errno = 0; ++ epoch = strtoull(source_date_epoch, &endptr, 10); ++ if((errno == ERANGE && (epoch == ULLONG_MAX || epoch == 0)) ++ || (errno != 0 && epoch == 0)) { ++ ERROR("Environment variable $SOURCE_DATE_EPOCH: " ++ "strtoull: %s\n", strerror(errno)); ++ EXIT_MKSQUASHFS(); ++ } ++ if(endptr == source_date_epoch) { ++ ERROR("Environment variable $SOURCE_DATE_EPOCH: " ++ "No digits were found: %s\n", endptr); ++ EXIT_MKSQUASHFS(); ++ } ++ if(*endptr != '\0') { ++ ERROR("Environment variable $SOURCE_DATE_EPOCH: " ++ "Trailing garbage: %s\n", endptr); ++ EXIT_MKSQUASHFS(); ++ } ++ if(epoch > ULONG_MAX) { ++ ERROR("Environment variable $SOURCE_DATE_EPOCH: " ++ "value must be smaller than or equal to " ++ "%lu but was found to be: %llu \n", ULONG_MAX, epoch); ++ EXIT_MKSQUASHFS(); ++ } ++ mkfs_fixed_time = (time_t)epoch; ++ } ++ + /* + * Some compressors may need the options to be checked for validity + * once all the options have been processed +@@ -5993,7 +6029,7 @@ printOptions: + sBlk.flags = SQUASHFS_MKFLAGS(noI, noD, noF, noX, no_fragments, + always_use_fragments, duplicate_checking, exportable, + no_xattrs, comp_opts); +- sBlk.mkfs_time = time(NULL); ++ sBlk.mkfs_time = mkfs_fixed_time != -1 ? mkfs_fixed_time : time(NULL); + + disable_info(); + +-- +2.17.0 + diff --git a/pkgs/tools/filesystems/squashfs/0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patch b/pkgs/tools/filesystems/squashfs/0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patch new file mode 100644 index 000000000000..5002375887fb --- /dev/null +++ b/pkgs/tools/filesystems/squashfs/0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patch @@ -0,0 +1,83 @@ +From 32a07d4156a281084c90a4b78affc8b0b32a26fc Mon Sep 17 00:00:00 2001 +From: intrigeri +Date: Mon, 21 Nov 2016 11:41:28 +0000 +Subject: [PATCH] If SOURCE_DATE_EPOCH is set, also clamp content timestamps + with that value. + +Based on a patch by Alexander Couzens posted on +https://sourceforge.net/p/squashfs/mailman/message/34673610/ +--- + squashfs-tools/mksquashfs.c | 15 ++++++++++++--- + 1 file changed, 12 insertions(+), 3 deletions(-) + +diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c +index b49e956..9f020bf 100644 +--- a/squashfs-tools/mksquashfs.c ++++ b/squashfs-tools/mksquashfs.c +@@ -137,6 +137,9 @@ unsigned int cache_bytes = 0, cache_size = 0, inode_count = 0; + /* inode lookup table */ + squashfs_inode *inode_lookup_table = NULL; + ++/* clamp all timestamps to SOURCE_DATE_EPOCH */ ++time_t content_clamp_time = -1; ++ + /* override filesystem creation time */ + time_t mkfs_fixed_time = -1; + +@@ -2246,6 +2249,8 @@ restat: + pathname_reader(dir_ent), strerror(errno)); + goto read_err; + } ++ if(content_clamp_time != -1 && buf2.st_mtime >= content_clamp_time) ++ buf2.st_mtime = content_clamp_time; + + if(read_size != buf2.st_size) { + close(file); +@@ -3101,7 +3106,7 @@ void dir_scan(squashfs_inode *inode, char *pathname, + buf.st_mode = S_IRWXU | S_IRWXG | S_IRWXO | S_IFDIR; + buf.st_uid = getuid(); + buf.st_gid = getgid(); +- buf.st_mtime = time(NULL); ++ buf.st_mtime = content_clamp_time != -1 ? content_clamp_time : time(NULL); + buf.st_dev = 0; + buf.st_ino = 0; + dir_ent->inode = lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0); +@@ -3127,6 +3115,8 @@ void dir_scan(squashfs_inode *inode, char *pathname, + /* source directory has disappeared? */ + BAD_ERROR("Cannot stat source directory %s because %s\n", + pathname, strerror(errno)); ++ if(content_clamp_time != -1 && buf.st_mtime >= content_clamp_time) ++ buf.st_mtime = content_clamp_time; + dir_ent->inode = lookup_inode(&buf); + } + +@@ -3365,6 +3372,8 @@ struct dir_info *dir_scan1(char *filename, char *subpath, + free_dir_entry(dir_ent); + continue; + } ++ if(content_clamp_time != -1 && buf.st_mtime >= content_clamp_time) ++ buf.st_mtime = content_clamp_time; + + if((buf.st_mode & S_IFMT) != S_IFREG && + (buf.st_mode & S_IFMT) != S_IFDIR && +@@ -3544,7 +3553,7 @@ void dir_scan2(struct dir_info *dir, struct pseudo *pseudo) + buf.st_gid = pseudo_ent->dev->gid; + buf.st_rdev = makedev(pseudo_ent->dev->major, + pseudo_ent->dev->minor); +- buf.st_mtime = time(NULL); ++ buf.st_mtime = content_clamp_time != -1 ? content_clamp_time : time(NULL); + buf.st_ino = pseudo_ino ++; + + if(pseudo_ent->dev->type == 'd') { +@@ -5674,7 +5683,7 @@ printOptions: + "%lu but was found to be: %llu \n", ULONG_MAX, epoch); + EXIT_MKSQUASHFS(); + } +- mkfs_fixed_time = (time_t)epoch; ++ mkfs_fixed_time = content_clamp_time = (time_t)epoch; + } + + /* +-- +2.17.0 + diff --git a/pkgs/tools/filesystems/squashfs/0003-remove-frag-deflator-thread.patch b/pkgs/tools/filesystems/squashfs/0003-remove-frag-deflator-thread.patch new file mode 100644 index 000000000000..4be4b96369a8 --- /dev/null +++ b/pkgs/tools/filesystems/squashfs/0003-remove-frag-deflator-thread.patch @@ -0,0 +1,220 @@ +From afc0c76a170bd17cbd29bbec6ae6d2227e398570 Mon Sep 17 00:00:00 2001 +From: Alexander Couzens +Date: Fri, 13 Jan 2017 22:00:37 +0100 +Subject: [PATCH] remove frag_deflator_thread + +frag_deflator_thread compress fragments. +Replace the deflator_thread with a function and +use the function instead of the to_frag queue. +--- + squashfs-tools/info.c | 5 --- + squashfs-tools/mksquashfs.c | 76 +++++++++++++------------------------ + squashfs-tools/mksquashfs.h | 2 +- + squashfs-tools/restore.c | 15 +------- + 4 files changed, 30 insertions(+), 68 deletions(-) + +diff --git a/squashfs-tools/info.c b/squashfs-tools/info.c +index 7968c77..028d578 100644 +--- a/squashfs-tools/info.c ++++ b/squashfs-tools/info.c +@@ -96,11 +96,6 @@ void dump_state() + printf("compressed block queue (deflate thread(s) -> main thread)\n"); + dump_seq_queue(to_main, 0); + +- printf("uncompressed packed fragment queue (main thread -> fragment" +- " deflate thread(s))\n"); +- dump_queue(to_frag); +- +- + printf("locked frag queue (compressed frags waiting while multi-block" + " file is written)\n"); + dump_queue(locked_fragment); +diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c +index cf48e40..cacf14c 100644 +--- a/squashfs-tools/mksquashfs.c ++++ b/squashfs-tools/mksquashfs.c +@@ -270,10 +270,10 @@ unsigned int sid_count = 0, suid_count = 0, sguid_count = 0; + struct cache *reader_buffer, *fragment_buffer, *reserve_cache; + struct cache *bwriter_buffer, *fwriter_buffer; + struct queue *to_reader, *to_deflate, *to_writer, *from_writer, +- *to_frag, *locked_fragment, *to_process_frag; ++ *locked_fragment, *to_process_frag; + struct seq_queue *to_main; + pthread_t reader_thread, writer_thread, main_thread; +-pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread; ++pthread_t *deflator_thread, *frag_thread; + pthread_t *restore_thread = NULL; + pthread_mutex_t fragment_mutex = PTHREAD_MUTEX_INITIALIZER; + pthread_mutex_t pos_mutex = PTHREAD_MUTEX_INITIALIZER; +@@ -323,7 +323,7 @@ struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth); + void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad); + unsigned short get_checksum_mem(char *buff, int bytes); + void check_usable_phys_mem(int total_mem); +- ++void frag_deflator(struct file_buffer *file_buffer); + + void prep_exit() + { +@@ -1540,7 +1540,7 @@ void write_fragment(struct file_buffer *fragment) + pthread_mutex_lock(&fragment_mutex); + fragment_table[fragment->block].unused = 0; + fragments_outstanding ++; +- queue_put(to_frag, fragment); ++ frag_deflator(fragment); + pthread_cleanup_pop(1); + } + +@@ -2412,51 +2412,34 @@ void *deflator(void *arg) + } + + +-void *frag_deflator(void *arg) ++void frag_deflator(struct file_buffer *file_buffer) + { +- void *stream = NULL; +- int res; + +- res = compressor_init(comp, &stream, block_size, 1); +- if(res) +- BAD_ERROR("frag_deflator:: compressor_init failed\n"); +- +- pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); +- +- while(1) { +- int c_byte, compressed_size; +- struct file_buffer *file_buffer = queue_get(to_frag); +- struct file_buffer *write_buffer = ++ int c_byte, compressed_size; ++ struct file_buffer *write_buffer = + cache_get(fwriter_buffer, file_buffer->block); + +- c_byte = mangle2(stream, write_buffer->data, file_buffer->data, +- file_buffer->size, block_size, noF, 1); +- compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte); +- write_buffer->size = compressed_size; +- pthread_mutex_lock(&fragment_mutex); +- if(fragments_locked == FALSE) { +- fragment_table[file_buffer->block].size = c_byte; +- fragment_table[file_buffer->block].start_block = bytes; +- write_buffer->block = bytes; +- bytes += compressed_size; +- fragments_outstanding --; +- queue_put(to_writer, write_buffer); +- pthread_mutex_unlock(&fragment_mutex); +- TRACE("Writing fragment %lld, uncompressed size %d, " +- "compressed size %d\n", file_buffer->block, +- file_buffer->size, compressed_size); +- } else { +- add_pending_fragment(write_buffer, c_byte, +- file_buffer->block); +- pthread_mutex_unlock(&fragment_mutex); +- } +- cache_block_put(file_buffer); ++ c_byte = mangle2(stream, write_buffer->data, file_buffer->data, ++ file_buffer->size, block_size, noF, 1); ++ compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte); ++ write_buffer->size = compressed_size; ++ if(fragments_locked == FALSE) { ++ fragment_table[file_buffer->block].size = c_byte; ++ fragment_table[file_buffer->block].start_block = bytes; ++ write_buffer->block = bytes; ++ bytes += compressed_size; ++ fragments_outstanding --; ++ queue_put(to_writer, write_buffer); ++ TRACE("Writing fragment %lld, uncompressed size %d, " ++ "compressed size %d\n", file_buffer->block, ++ file_buffer->size, compressed_size); ++ } else { ++ add_pending_fragment(write_buffer, c_byte, ++ file_buffer->block); + } +- +- pthread_cleanup_pop(0); ++ cache_block_put(file_buffer); + } + +- + struct file_buffer *get_file_buffer() + { + struct file_buffer *file_buffer = seq_queue_get(to_main); +@@ -4257,19 +4240,17 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq, + multiply_overflow(processors * 3, sizeof(pthread_t))) + BAD_ERROR("Processors too large\n"); + +- deflator_thread = malloc(processors * 3 * sizeof(pthread_t)); ++ deflator_thread = malloc(processors * 2 * sizeof(pthread_t)); + if(deflator_thread == NULL) + MEM_ERROR(); + +- frag_deflator_thread = &deflator_thread[processors]; +- frag_thread = &frag_deflator_thread[processors]; ++ frag_thread = &deflator_thread[processors]; + + to_reader = queue_init(1); + to_deflate = queue_init(reader_size); + to_process_frag = queue_init(reader_size); + to_writer = queue_init(bwriter_size + fwriter_size); + from_writer = queue_init(1); +- to_frag = queue_init(fragment_size); + locked_fragment = queue_init(fragment_size); + to_main = seq_queue_init(); + reader_buffer = cache_init(block_size, reader_size, 0, 0); +@@ -4285,9 +4266,6 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq, + for(i = 0; i < processors; i++) { + if(pthread_create(&deflator_thread[i], NULL, deflator, NULL)) + BAD_ERROR("Failed to create thread\n"); +- if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator, +- NULL) != 0) +- BAD_ERROR("Failed to create thread\n"); + if(pthread_create(&frag_thread[i], NULL, frag_thrd, + (void *) destination_file) != 0) + BAD_ERROR("Failed to create thread\n"); +diff --git a/squashfs-tools/mksquashfs.h b/squashfs-tools/mksquashfs.h +index 55708a3..dc5bde4 100644 +--- a/squashfs-tools/mksquashfs.h ++++ b/squashfs-tools/mksquashfs.h +@@ -135,7 +135,7 @@ struct append_file { + extern struct cache *reader_buffer, *fragment_buffer, *reserve_cache; + struct cache *bwriter_buffer, *fwriter_buffer; + extern struct queue *to_reader, *to_deflate, *to_writer, *from_writer, +- *to_frag, *locked_fragment, *to_process_frag; ++ *locked_fragment, *to_process_frag; + extern struct append_file **file_mapping; + extern struct seq_queue *to_main; + extern pthread_mutex_t fragment_mutex, dup_mutex; +diff --git a/squashfs-tools/restore.c b/squashfs-tools/restore.c +index 5e336b3..a7aaf2e 100644 +--- a/squashfs-tools/restore.c ++++ b/squashfs-tools/restore.c +@@ -47,8 +47,8 @@ + #define TRUE 1 + + extern pthread_t reader_thread, writer_thread, main_thread; +-extern pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread; +-extern struct queue *to_deflate, *to_writer, *to_frag, *to_process_frag; ++extern pthread_t *deflator_thread, *frag_thread; ++extern struct queue *to_deflate, *to_writer, *to_process_frag; + extern struct seq_queue *to_main; + extern void restorefs(); + extern int processors; +@@ -120,17 +120,6 @@ void *restore_thrd(void *arg) + pthread_cancel(main_thread); + pthread_join(main_thread, NULL); + +- /* then flush the main thread to fragment deflator thread(s) +- * queue. The fragment deflator thread(s) will idle +- */ +- queue_flush(to_frag); +- +- /* now kill the fragment deflator thread(s) */ +- for(i = 0; i < processors; i++) +- pthread_cancel(frag_deflator_thread[i]); +- for(i = 0; i < processors; i++) +- pthread_join(frag_deflator_thread[i], NULL); +- + /* + * then flush the main thread/fragment deflator thread(s) + * to writer thread queue. The writer thread will idle +-- +2.17.0 + diff --git a/pkgs/tools/filesystems/squashfs/default.nix b/pkgs/tools/filesystems/squashfs/default.nix index a2fc5bc3d40b..08951113b3b7 100644 --- a/pkgs/tools/filesystems/squashfs/default.nix +++ b/pkgs/tools/filesystems/squashfs/default.nix @@ -15,6 +15,15 @@ stdenv.mkDerivation rec { rev = "9c1db6d13a51a2e009f0027ef336ce03624eac0d"; }; + # These patches ensures that mksquashfs output is reproducible. + # See also https://reproducible-builds.org/docs/system-images/ + # and https://github.com/NixOS/nixpkgs/issues/40144. + patches = [ + ./0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patch + ./0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patch + ./0003-remove-frag-deflator-thread.patch + ]; + buildInputs = [ zlib xz ] ++ stdenv.lib.optional lz4Support lz4;