From 282f2414598ec8b675aa922f8949718c6f00e61b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Mon, 1 Oct 2018 21:08:30 +0200 Subject: [PATCH 01/16] Missing dependency in offsets calculation (could cause race condition when building with -j) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1f315e3..eade1ba 100644 --- a/Makefile +++ b/Makefile @@ -219,7 +219,7 @@ ${eval ${call offset,bytes_fat12_end, $${bytes_fat12_start} + $${bytes_fa ${eval ${call offset,bytes_gpt_mirror_size, $${sectors_gpt_mirror_size} + $${sector_size}, sectors_gpt_mirror_size,}} ${eval ${call offset,bytes_gpt_mirror_end, $${bytes_fat12_end} + $${bytes_gpt_mirror_size}, bytes_fat12_end bytes_gpt_mirror_size,}} ${eval ${call offset,bytes_gpt_mirror_start, $${bytes_gpt_mirror_end} - $${bytes_gpt_mirror_size}, bytes_gpt_mirror_end bytes_gpt_mirror_size,}} -${eval ${call offset,bytes_zip_end, $${bytes_os_size},, }} +${eval ${call offset,bytes_zip_end, $${bytes_os_size}, bytes_os_size,}} os_fat12_partition = "$@@@${bytes_fat12_start}" build/os.fat12: build/os.zip ${dep_bytes_fat12_size} ${dep_bytes_fat12_start} ${dep_sectors_os_size} build/check_makefile -- 2.34.1 From cd866fb6e70e5809d742afa12e302a40bea6f4e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Sun, 26 Aug 2018 18:36:33 +0200 Subject: [PATCH 02/16] Closes #42: Reproducible builds 232 initial number of different lines in the hexadecimal dump. 160 (recursive touch in iso_files) 108 (touch os.32k) 46 (faketime on mkisofs) 38 (faketime on mformat) 32 (faketime on mcopy) 0 (explicit partition and disk GUIDs) --- .travis.yml | 1 + Makefile | 76 ++++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 59 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9794fa3..6f4e54c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,7 @@ addons: - nasm - mtools - genisoimage + - faketime matrix: include: diff --git a/Makefile b/Makefile index eade1ba..807db5a 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,9 @@ tests_emu = test/qemu-system-i386-floppy test/qemu-system-i386-cdrom test/qemu-s tests_requiring_sudo = test/fat12_mount test/iso_mount tests_noemu = test/zip test/os.reasm test/sizes test/fat12_contents +commit_timestamp = "$$(git log -1 --pretty=format:%ad --date=iso8601-strict)" +commit_faketime = "$$(git log -1 --pretty=format:%ad --date=format:"%Y-%m-%d %H:%m:%S")" + offset_names = bytes_os_size \ bytes_mbr_start \ bytes_mbr_end \ @@ -86,6 +89,14 @@ built_files = ${os_filename} \ ${tests_emu:test/%=deploy-screenshots/%.png} \ ${tests_emu:test/%=deploy-screenshots/%-anim.gif} +# Temporary copies used to adjust timestamps for reproducible builds. +# These are normally created and deleted within a single target, but +# could remain if make is interrupted during the build. +temp_files = build/iso_files.tmp/os.zip \ + build/iso_files.tmp/boot/iso_boot.sys +temp_directories = build/iso_files.tmp \ + build/iso_files.tmp/boot/ + built_directories = build/iso_files/boot build/iso_files build/offsets build/mnt_fat12 build/mnt_iso build/test_pass deploy-screenshots more_built_directories = ${built_directories} build @@ -150,7 +161,9 @@ build/os.32k: example-os/os.asm build/check_makefile nasm -w+macro-params -w+macro-selfref -w+orphan-labels -w+gnu-elf-extensions -o $@ $< build/os.iso: build/iso_files/os.zip build/iso_files/boot/iso_boot.sys build/check_makefile - mkisofs \ + cp -a -- build/iso_files build/iso_files.tmp + find build/iso_files.tmp -depth -exec touch -d ${commit_timestamp} '{}' ';' + faketime -f ${commit_faketime} mkisofs \ --input-charset utf-8 \ -rock \ -joliet \ @@ -160,7 +173,11 @@ build/os.iso: build/iso_files/os.zip build/iso_files/boot/iso_boot.sys build/che -boot-load-size 4 \ -pad \ -output $@ \ - ./build/iso_files/ + ./build/iso_files.tmp/ + rm -- build/iso_files.tmp/os.zip \ + build/iso_files.tmp/boot/iso_boot.sys + rmdir build/iso_files.tmp/boot/ + rmdir build/iso_files.tmp/ # Layout: # MBR; GPT; UNIX sh & MS-DOS batch scripts; ISO9660; FAT12; GPT mirror; ZIP @@ -224,12 +241,12 @@ ${eval ${call offset,bytes_zip_end, $${bytes_os_size}, os_fat12_partition = "$@@@${bytes_fat12_start}" build/os.fat12: build/os.zip ${dep_bytes_fat12_size} ${dep_bytes_fat12_start} ${dep_sectors_os_size} build/check_makefile set -x; dd if=/dev/zero bs=${sector_size} count=${sectors_os_size} of=$@ - set -x; mformat -v "Example OS" \ + faketime -f ${commit_faketime} mformat -v "Example OS" \ -T ${sectors_fat12_size} \ -h ${os_floppy_chs_h} \ -s ${os_floppy_chs_s} \ -i ${os_fat12_partition} - set -x; mcopy -i ${os_fat12_partition} build/os.zip "::os.zip" + faketime -f ${commit_faketime} mcopy -i ${os_fat12_partition} $< "::os.zip" build/iso_files/os.zip: build/os.zip build/check_makefile # TODO: make it so that the various file formats are mutual quines: @@ -245,7 +262,14 @@ build/iso_files/boot/iso_boot.sys: build/os.32k build/check_makefile dd if=$< bs=512 count=4 of=$@ build/os.zip: build/os.32k build/check_makefile - zip $@ $< +# We copy os.32k and alter its timestamp to ensure reproducible +# builds. + mkdir -p build/os.32k.tmp + cp -a $< build/os.32k.tmp/os.32k + touch -d ${commit_timestamp} build/os.32k.tmp/os.32k + (cd build/os.32k.tmp/ && zip -X ../../build/os.zip os.32k) + rm build/os.32k.tmp/os.32k + rmdir build/os.32k.tmp build/os.zip.adjusted: build/os.zip ${dep_bytes_zip_start} build/check_makefile # TODO: the ZIP file can end with a variable-length comment, this would allow us to hide the GPT mirrors. @@ -253,6 +277,13 @@ build/os.zip.adjusted: build/os.zip ${dep_bytes_zip_start} build/check_makefile cat $< >> $@ zip --adjust-sfx $@ +gdisk_pipe_commands_slowly=while read str; do echo "$$str"; printf "\033[1;33m%s\033[m\n" "$$str" >&2; sleep 0.01; done + +commit_hash_as_guid=$(git log -1 --pretty=format:%H | sed -e 's/^\(.\{8\}\)\(.\{4\}\)\(.\{4\}\)\(.\{4\}\)\(.\{12\}\).*$$/\1-\2-\3-\4-\5/' | tr '[:lower:]' '[:upper:]') +git_dirty=test -n "$$(git diff --shortstat)" +gpt_disk_guid=${commit_hash_as_guid}$$(if $git_dirty; then printf '0'; else printf '2'; fi) +gpt_partition_guid=${commit_hash_as_guid}$$(if $git_dirty; then printf '1'; else printf '3'; fi) + ${os_filename}: build/os.32k build/os.iso build/os.fat12 build/os.zip.adjusted \ ${dep_bytes_header_32k_start} \ ${dep_bytes_header_32k_size} \ @@ -281,13 +312,21 @@ ${os_filename}: build/os.32k build/os.iso build/os.fat12 build/os.zip.adjusted \ # * Recovery and transformation options, make Hybrid, # * add GPT partition #1 to the hybrid MBR, do Not put the EFI partition first, # * MBR partition type=0x01, bootable=Yes, do Not add extra partitions, -# * Print GPT, print MBR, Write, Proceed. +# * back to Main menu, +# * eXpert mode, +# * Change partition GUID, the GUID itself, +# * change disk GUID, the GUID itself, +# * Print GPT, print prOtective MBR, Write, Proceed. (if test "$$(uname -o)" = "Cygwin"; then printf "Y\n"; fi; \ - printf "d\nx\nl\n1\nm\nn\n1\n${sectors_fat12_start}\n${sectors_fat12_size}\n0700\n"; \ + printf "d\nx\nl\n1\nm\n"; \ + printf "n\n1\n${sectors_fat12_start}\n${sectors_fat12_size}\n0700\n"; \ printf "r\nh\n"; \ printf "1\nN\n"; \ printf "01\nY\nN\n"; \ - printf "p\no\nw\nY\n") | while read str; do echo "$$str"; printf "\033[1;33m%s\033[m\n" "$$str" >&2; sleep 0.01; done | gdisk $@ + printf "x\n"; \ + printf "g\n${gpt_disk_guid}\n"; \ + printf "c\n${gpt_partition_guid}\n"; \ + printf "p\no\nw\nY\n") | ${gdisk_pipe_commands_slowly} | gdisk $@ # Inject MS-DOS newlines (CR+LF) and comments (":: ") in the GUID field of unused partition table entries, # so that the part that is to be skipped by MS-DOS does not form a line longer than the MS-DOS maximum # line length (8192 excluding CR+LF). $i below is the partition entry number, starting from 1 @@ -305,11 +344,12 @@ build/os.file: ${os_filename} build/check_makefile build/os.gdisk: ${os_filename} build/check_makefile # gdisk commands: -# * Print partition table -# * Recovery and transformation options -# * print prOtective MBR table -# * Quit - printf 'p\nr\no\nq\n' | gdisk $< | tee $@ +# * eXpert mode +# * Print partition table +#  * print detailed Information about the (only) partition +# * print prOtective MBR table +# * Quit + printf '2\nx\np\ni\nr\no\nq\n' | ${gdisk_pipe_commands_slowly} | gdisk $< | tee $@ build/os.offsets.hex: ${offset_names:%=build/offsets/%.hex} build/check_makefile grep '^' ${offset_names:%=build/offsets/%.hex} | sed -e 's/:/: 0x/' | column -t > $@ @@ -346,8 +386,8 @@ build/test_pass/noemu_%.reasm build/%.reasm: build/%.reasm.asm ${os_filename} ut .PHONY: clean clean: build/check_makefile - rm -f ${built_files} - for d in $$(echo ${more_built_directories} | tr ' ' '\n' | sort --reverse); do \ + rm -f ${built_files} ${temp_files} + for d in $$(echo ${more_built_directories} ${temp_directories} | tr ' ' '\n' | sort --reverse); do \ if test -e "$$d"; then \ rmdir "$$d"; \ fi; \ @@ -409,7 +449,7 @@ build/test_pass/sudo_iso_mount: ${os_filename} build/check_makefile | build/mnt_ grep '^' build/offsets/* # debug failure to mount the ISO9660 filesystem (sudo mount -o loop,ro $< build/mnt_iso) || true dmesg | tail # debug failure to mount the ISO9660 filesystem - hexdump -C os.bat + hexdump -C ${os_filename} ls -l build/mnt_iso | grep os.zip sudo umount build/mnt_iso sudo mount -o loop,ro $< build/mnt_iso @@ -425,7 +465,7 @@ test/macos-sh-x11: sudo chmod a+rwxt /tmp/.X11-unix xvfb :42 & \ sleep 5; \ - DISPLAY=:42 xterm -e ./os.bat & \ + DISPLAY=:42 xterm -e ./${os_filename} & \ sleep 5; \ # DISPLAY=:42 import -window root deploy-screenshots/macos-sh-x11.png screencapture deploy-screenshots/macos-sh-x11-screencapture.png @@ -434,7 +474,7 @@ test/macos-sh-x11: test/macos-sh: build/check_makefile \ build/checkerboard_1024x768.png \ | deploy-screenshots - osascript -e 'tell app "Terminal" to do script "'"$$PWD"'/os.bat"' + osascript -e 'tell app "Terminal" to do script "'"$$PWD"'/${os_filename}"' sleep 2 osascript -e 'tell app "Terminal" to activate' sleep 5 -- 2.34.1 From 01d3ae7544b186d7367be1974e25cf4d63aa02f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Sun, 26 Aug 2018 20:22:09 +0200 Subject: [PATCH 03/16] Made the build directory configurable --- Makefile | 315 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 167 insertions(+), 148 deletions(-) diff --git a/Makefile b/Makefile index 807db5a..5870d21 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,8 @@ SHELL = bash -euET -o pipefail -c .SECONDEXPANSION: os_filename = os.bat +BUILD_DIR = build +bld = ${BUILD_DIR} tests_emu = test/qemu-system-i386-floppy test/qemu-system-i386-cdrom test/qemu-system-arm test/virtualbox test/bochs test/gui-sh test/dosbox tests_requiring_sudo = test/fat12_mount test/iso_mount tests_noemu = test/zip test/os.reasm test/sizes test/fat12_contents @@ -42,63 +44,69 @@ more_offset_names = ${offset_names} \ tracks_os_size \ tracks_zip_size -more_offset_dec = ${more_offset_names:%=build/offsets/%.dec} -more_offset_hex = ${more_offset_names:%=build/offsets/%.hex} +more_offset_dec = ${more_offset_names:%=${bld}/offsets/%.dec} +more_offset_hex = ${more_offset_names:%=${bld}/offsets/%.hex} # + os.arm.disasm # + os.reasm.disasm built_files = ${os_filename} \ - build/check_makefile \ - build/check_makefile_targets \ - build/check_makefile_w_arnings \ - build/checkerboard_800x600.xbm \ - build/checkerboard_1024x768.png \ - build/makefile_built_directories \ - build/makefile_built_files \ - build/makefile_database \ - build/makefile_database_files \ - build/makefile_file_targets \ - build/makefile_non_file_targets \ - build/makefile_phony \ - build/makefile_targets \ - build/makefile_w_arnings \ - build/os.ndisasm.disasm \ - build/os.reasm.asm \ - build/os.reasm \ - build/os.file \ - build/os.gdisk \ - build/os.zip \ - build/os.zip.adjusted \ - build/os.iso \ - build/os.32k \ - build/os.fat12 \ - build/os.offsets.hex \ - build/os.offsets.dec \ - build/os.hex_with_offsets \ - build/iso_files/os.zip \ - build/iso_files/boot/iso_boot.sys \ - build/bochsrc \ - build/bochscontinue \ - build/twm_cfg \ - build/virtualbox.img \ + ${bld}/check_makefile \ + ${bld}/check_makefile_targets \ + ${bld}/check_makefile_w_arnings \ + ${bld}/checkerboard_800x600.xbm \ + ${bld}/checkerboard_1024x768.png \ + ${bld}/makefile_built_directories \ + ${bld}/makefile_built_files \ + ${bld}/makefile_database \ + ${bld}/makefile_database_files \ + ${bld}/makefile_file_targets \ + ${bld}/makefile_non_file_targets \ + ${bld}/makefile_phony \ + ${bld}/makefile_targets \ + ${bld}/makefile_w_arnings \ + ${bld}/os.ndisasm.disasm \ + ${bld}/os.reasm.asm \ + ${bld}/os.reasm \ + ${bld}/os.file \ + ${bld}/os.gdisk \ + ${bld}/os.zip \ + ${bld}/os.zip.adjusted \ + ${bld}/os.iso \ + ${bld}/os.32k \ + ${bld}/os.fat12 \ + ${bld}/os.offsets.hex \ + ${bld}/os.offsets.dec \ + ${bld}/os.hex_with_offsets \ + ${bld}/iso_files/os.zip \ + ${bld}/iso_files/boot/iso_boot.sys \ + ${bld}/bochsrc \ + ${bld}/bochscontinue \ + ${bld}/twm_cfg \ + ${bld}/virtualbox.img \ ${more_offset_dec} \ ${more_offset_hex} \ - ${tests_emu:test/%=build/test_pass/emu_%} \ - ${tests_noemu:test/%=build/test_pass/noemu_%} \ - ${tests_requiring_sudo:test/%=build/test_pass/sudo_%} \ + ${tests_emu:test/%=${bld}/test_pass/emu_%} \ + ${tests_noemu:test/%=${bld}/test_pass/noemu_%} \ + ${tests_requiring_sudo:test/%=${bld}/test_pass/sudo_%} \ ${tests_emu:test/%=deploy-screenshots/%.png} \ ${tests_emu:test/%=deploy-screenshots/%-anim.gif} # Temporary copies used to adjust timestamps for reproducible builds. # These are normally created and deleted within a single target, but # could remain if make is interrupted during the build. -temp_files = build/iso_files.tmp/os.zip \ - build/iso_files.tmp/boot/iso_boot.sys -temp_directories = build/iso_files.tmp \ - build/iso_files.tmp/boot/ +temp_files = ${bld}/iso_files.tmp/os.zip \ + ${bld}/iso_files.tmp/boot/iso_boot.sys +temp_directories = ${bld}/iso_files.tmp \ + ${bld}/iso_files.tmp/boot/ -built_directories = build/iso_files/boot build/iso_files build/offsets build/mnt_fat12 build/mnt_iso build/test_pass deploy-screenshots -more_built_directories = ${built_directories} build +built_directories = ${bld}/iso_files/boot \ + ${bld}/iso_files \ + ${bld}/offsets \ + ${bld}/mnt_fat12 \ + ${bld}/mnt_iso \ + ${bld}/test_pass \ + deploy-screenshots +more_built_directories = ${built_directories} ${bld} os_image_size_kb = 1440 os_partition_start_sectors = 3 @@ -109,60 +117,71 @@ os_floppy_chs_s = 9 .PHONY: all # all: os.arm.disasm -all: ${os_filename} build/os.ndisasm.disasm build/os.reasm.asm build/os.file build/os.gdisk build/os.offsets.hex build/os.offsets.dec build/os.hex_with_offsets .gitignore build/check_makefile ${more_offset_dec} ${more_offset_hex} +all: ${os_filename} \ + ${bld}/os.ndisasm.disasm \ + ${bld}/os.reasm.asm \ + ${bld}/os.file \ + ${bld}/os.gdisk \ + ${bld}/os.offsets.hex \ + ${bld}/os.offsets.dec \ + ${bld}/os.hex_with_offsets \ + .gitignore \ + ${bld}/check_makefile \ + ${more_offset_dec} \ + ${more_offset_hex} -build/makefile_w_arnings: | $${@D} +${bld}/makefile_w_arnings: | $${@D} ${built_files}: | $${@D} -build/makefile_w_arnings: Makefile - @unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; make -n --warn-undefined-variables test 2>$@ 1>/dev/null || make -n --warn-undefined-variables test +${bld}/makefile_w_arnings: Makefile + @unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; make -n --warn-undefined-variables BUILD_DIR=${BUILD_DIR} test 2>$@ 1>/dev/null || make -n --warn-undefined-variables test -# Check that the file build/makefile_w_arnings is present, and that it does not contain the string "warn". -build/check_makefile_w_arnings: build/makefile_w_arnings - @cat build/makefile_w_arnings > /dev/null && (! grep -i warn $<) && touch $@ +# Check that the file ${bld}/makefile_w_arnings is present, and that it does not contain the string "warn". +${bld}/check_makefile_w_arnings: ${bld}/makefile_w_arnings + @cat ${bld}/makefile_w_arnings > /dev/null && (! grep -i warn $<) && touch $@ # Check that the declared list of built files matches the list of targets extracted from the Makefile. -build/check_makefile_targets: build/makefile_built_files build/makefile_file_targets build/check_makefile_w_arnings - @diff build/makefile_built_files build/makefile_file_targets && touch $@ +${bld}/check_makefile_targets: ${bld}/makefile_built_files ${bld}/makefile_file_targets ${bld}/check_makefile_w_arnings + @diff ${bld}/makefile_built_files ${bld}/makefile_file_targets && touch $@ -build/check_makefile: build/check_makefile_w_arnings build/check_makefile_targets +${bld}/check_makefile: ${bld}/check_makefile_w_arnings ${bld}/check_makefile_targets @touch $@ -build/makefile_database: Makefile build/check_makefile_w_arnings - @unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; make -rpn | sed -n -e '/^# Make data base,/,$$p' > $@ +${bld}/makefile_database: Makefile ${bld}/check_makefile_w_arnings + @unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; make -rpn BUILD_DIR=${BUILD_DIR} | sed -n -e '/^# Make data base,/,$$p' > $@ -build/makefile_database_files: build/makefile_database build/check_makefile_w_arnings +${bld}/makefile_database_files: ${bld}/makefile_database ${bld}/check_makefile_w_arnings @sed -n -e '/^# Files$$/,/^# files hash-table stats:$$/p' $< > $@ -build/makefile_built_directories: build/check_makefile_w_arnings +${bld}/makefile_built_directories: ${bld}/check_makefile_w_arnings @echo ${more_built_directories} | tr ' ' '\n' | grep -v '^[[:space:]]*$$' | sort > $@ -build/makefile_built_files: build/check_makefile_w_arnings +${bld}/makefile_built_files: ${bld}/check_makefile_w_arnings @echo ${built_files} | tr ' ' '\n' | grep -v '^[[:space:]]*$$' | sort > $@ -build/makefile_phony: build/makefile_database_files build/check_makefile_w_arnings +${bld}/makefile_phony: ${bld}/makefile_database_files ${bld}/check_makefile_w_arnings @sed -n -e 's/^\.PHONY: \(.*\)$$/\1/p' $< | tr ' ' '\n' | grep -v '^[[:space:]]*$$' | sort > $@ -build/makefile_targets: build/makefile_database_files build/check_makefile_w_arnings +${bld}/makefile_targets: ${bld}/makefile_database_files ${bld}/check_makefile_w_arnings @grep -E -v '^([[:space:]]|#|\.|$$|^[^:]*:$$)' $< | grep '^[^ :]*:' | sed -e 's|^\([^:]*\):.*$$|\1|' | sort > $@ -build/makefile_non_file_targets: build/makefile_phony build/makefile_built_directories build/check_makefile_w_arnings - @cat build/makefile_phony build/makefile_built_directories | sort > $@ +${bld}/makefile_non_file_targets: ${bld}/makefile_phony ${bld}/makefile_built_directories ${bld}/check_makefile_w_arnings + @cat ${bld}/makefile_phony ${bld}/makefile_built_directories | sort > $@ -build/makefile_file_targets: build/makefile_non_file_targets build/makefile_targets build/check_makefile_w_arnings - @comm -23 build/makefile_targets build/makefile_non_file_targets > $@ +${bld}/makefile_file_targets: ${bld}/makefile_non_file_targets ${bld}/makefile_targets ${bld}/check_makefile_w_arnings + @comm -23 ${bld}/makefile_targets ${bld}/makefile_non_file_targets > $@ -${built_directories}: build/check_makefile +${built_directories}: ${bld}/check_makefile ${more_built_directories}: Makefile mkdir -p $@ && touch $@ # 32k header of the ISO9660 image -build/os.32k: example-os/os.asm build/check_makefile +${bld}/os.32k: example-os/os.asm ${bld}/check_makefile nasm -w+macro-params -w+macro-selfref -w+orphan-labels -w+gnu-elf-extensions -o $@ $< -build/os.iso: build/iso_files/os.zip build/iso_files/boot/iso_boot.sys build/check_makefile - cp -a -- build/iso_files build/iso_files.tmp - find build/iso_files.tmp -depth -exec touch -d ${commit_timestamp} '{}' ';' +${bld}/os.iso: ${bld}/iso_files/os.zip ${bld}/iso_files/boot/iso_boot.sys ${bld}/check_makefile + cp -a -- ${bld}/iso_files ${bld}/iso_files.tmp + find ${bld}/iso_files.tmp -depth -exec touch -d ${commit_timestamp} '{}' ';' faketime -f ${commit_faketime} mkisofs \ --input-charset utf-8 \ -rock \ @@ -173,21 +192,21 @@ build/os.iso: build/iso_files/os.zip build/iso_files/boot/iso_boot.sys build/che -boot-load-size 4 \ -pad \ -output $@ \ - ./build/iso_files.tmp/ - rm -- build/iso_files.tmp/os.zip \ - build/iso_files.tmp/boot/iso_boot.sys - rmdir build/iso_files.tmp/boot/ - rmdir build/iso_files.tmp/ + ./${bld}/iso_files.tmp/ + rm -- ${bld}/iso_files.tmp/os.zip \ + ${bld}/iso_files.tmp/boot/iso_boot.sys + rmdir ${bld}/iso_files.tmp/boot/ + rmdir ${bld}/iso_files.tmp/ # Layout: # MBR; GPT; UNIX sh & MS-DOS batch scripts; ISO9660; FAT12; GPT mirror; ZIP define offset tmp_${1} = ${3} -build/offsets/${1}.dec: $${tmp_${1}:%=build/offsets/%.dec} ${4} build/check_makefile +${bld}/offsets/${1}.dec: $${tmp_${1}:%=${bld}/offsets/%.dec} ${4} ${bld}/check_makefile echo $$$$(( ${2} )) | tee $$@ -${1} = $$$$(cat build/offsets/${1}.dec) -dep_${1} = build/offsets/${1}.dec +${1} = $$$$(cat ${bld}/offsets/${1}.dec) +dep_${1} = ${bld}/offsets/${1}.dec endef define div_round_up @@ -201,12 +220,12 @@ ${eval ${call offset,sectors_os_size, $${bytes_os_size} / $${sector_size}, ${eval ${call offset,tracks_os_size, $${sectors_os_size} / $${os_floppy_chs_s}, sectors_os_size,}} # round up -${eval ${call offset,bytes_iso_size, $$$$(utils/file-length.sh -c build/os.iso), ,build/os.iso}} +${eval ${call offset,bytes_iso_size, $$$$(utils/file-length.sh -c ${bld}/os.iso), ,${bld}/os.iso}} ${eval ${call offset,sectors_iso_size, ${call div_round_up,$${bytes_iso_size},$${sector_size}}, bytes_iso_size,}} ${eval ${call offset,tracks_iso_size, ${call div_round_up,$${sectors_iso_size},$${os_floppy_chs_s}}, sectors_iso_size,}} # round up -${eval ${call offset,bytes_zip_size, $$$$(utils/file-length.sh -c build/os.zip), ,build/os.zip}} +${eval ${call offset,bytes_zip_size, $$$$(utils/file-length.sh -c ${bld}/os.zip), ,${bld}/os.zip}} ${eval ${call offset,sectors_zip_size, ${call div_round_up,$${bytes_zip_size},$${sector_size}}, bytes_zip_size,}} ${eval ${call offset,tracks_zip_size, ${call div_round_up,$${sectors_zip_size},$${os_floppy_chs_s}}, sectors_zip_size,}} @@ -239,7 +258,7 @@ ${eval ${call offset,bytes_gpt_mirror_start, $${bytes_gpt_mirror_end} - $${bytes ${eval ${call offset,bytes_zip_end, $${bytes_os_size}, bytes_os_size,}} os_fat12_partition = "$@@@${bytes_fat12_start}" -build/os.fat12: build/os.zip ${dep_bytes_fat12_size} ${dep_bytes_fat12_start} ${dep_sectors_os_size} build/check_makefile +${bld}/os.fat12: ${bld}/os.zip ${dep_bytes_fat12_size} ${dep_bytes_fat12_start} ${dep_sectors_os_size} ${bld}/check_makefile set -x; dd if=/dev/zero bs=${sector_size} count=${sectors_os_size} of=$@ faketime -f ${commit_faketime} mformat -v "Example OS" \ -T ${sectors_fat12_size} \ @@ -248,7 +267,7 @@ build/os.fat12: build/os.zip ${dep_bytes_fat12_size} ${dep_bytes_fat12_start} ${ -i ${os_fat12_partition} faketime -f ${commit_faketime} mcopy -i ${os_fat12_partition} $< "::os.zip" -build/iso_files/os.zip: build/os.zip build/check_makefile +${bld}/iso_files/os.zip: ${bld}/os.zip ${bld}/check_makefile # TODO: make it so that the various file formats are mutual quines: # * the ISO should contain the original file # * the ZIP should contain the original file @@ -256,22 +275,22 @@ build/iso_files/os.zip: build/os.zip build/check_makefile cp $< $@ # 4 sectors loaded when booting from optical media (CD-ROM, …): -build/iso_files/boot/iso_boot.sys: build/os.32k build/check_makefile +${bld}/iso_files/boot/iso_boot.sys: ${bld}/os.32k ${bld}/check_makefile # TODO: this copy of the (or alternate) bootsector should contain a Boot Information Table, # see https://wiki.osdev.org/El-Torito#A_BareBones_Boot_Image_with_Boot_Information_Table dd if=$< bs=512 count=4 of=$@ -build/os.zip: build/os.32k build/check_makefile +${bld}/os.zip: ${bld}/os.32k ${bld}/check_makefile # We copy os.32k and alter its timestamp to ensure reproducible # builds. - mkdir -p build/os.32k.tmp - cp -a $< build/os.32k.tmp/os.32k - touch -d ${commit_timestamp} build/os.32k.tmp/os.32k - (cd build/os.32k.tmp/ && zip -X ../../build/os.zip os.32k) - rm build/os.32k.tmp/os.32k - rmdir build/os.32k.tmp + mkdir -p ${bld}/os.32k.tmp + cp -a $< ${bld}/os.32k.tmp/os.32k + touch -d ${commit_timestamp} ${bld}/os.32k.tmp/os.32k + (cd ${bld}/os.32k.tmp/ && zip -X ../os.zip os.32k) + rm ${bld}/os.32k.tmp/os.32k + rmdir ${bld}/os.32k.tmp -build/os.zip.adjusted: build/os.zip ${dep_bytes_zip_start} build/check_makefile +${bld}/os.zip.adjusted: ${bld}/os.zip ${dep_bytes_zip_start} ${bld}/check_makefile # TODO: the ZIP file can end with a variable-length comment, this would allow us to hide the GPT mirrors. set -x; dd if=/dev/zero bs=1 count=${bytes_zip_start} of=$@ cat $< >> $@ @@ -284,7 +303,7 @@ git_dirty=test -n "$$(git diff --shortstat)" gpt_disk_guid=${commit_hash_as_guid}$$(if $git_dirty; then printf '0'; else printf '2'; fi) gpt_partition_guid=${commit_hash_as_guid}$$(if $git_dirty; then printf '1'; else printf '3'; fi) -${os_filename}: build/os.32k build/os.iso build/os.fat12 build/os.zip.adjusted \ +${os_filename}: ${bld}/os.32k ${bld}/os.iso ${bld}/os.fat12 ${bld}/os.zip.adjusted \ ${dep_bytes_header_32k_start} \ ${dep_bytes_header_32k_size} \ ${dep_bytes_fat12_start} \ @@ -294,14 +313,14 @@ ${os_filename}: build/os.32k build/os.iso build/os.fat12 build/os.zip.adjusted \ ${dep_sectors_fat12_start} \ ${dep_sectors_fat12_size} \ ${dep_bytes_zip_start} \ - build/check_makefile + ${bld}/check_makefile rm -f $@ # start with the .iso - cp build/os.iso $@ + cp ${bld}/os.iso $@ # splice in the first 32k (bootsector and partition table) - set -x; dd skip=${bytes_header_32k_start} seek=${bytes_header_32k_start} bs=1 count=${bytes_header_32k_size} conv=notrunc if=build/os.32k of=$@ + set -x; dd skip=${bytes_header_32k_start} seek=${bytes_header_32k_start} bs=1 count=${bytes_header_32k_size} conv=notrunc if=${bld}/os.32k of=$@ # splice in fat12 - set -x; dd skip=${bytes_fat12_start} seek=${bytes_fat12_start} bs=1 count=${bytes_fat12_size} conv=notrunc if=build/os.fat12 of=$@ + set -x; dd skip=${bytes_fat12_start} seek=${bytes_fat12_start} bs=1 count=${bytes_fat12_size} conv=notrunc if=${bld}/os.fat12 of=$@ # pad with zeroes to prepare for GPT table set -x; dd if=/dev/zero seek=$$((${bytes_gpt_mirror_end} - 1 )) bs=1 count=1 conv=notrunc of=$@ # patch the partition table @@ -336,13 +355,13 @@ ${os_filename}: build/os.32k build/os.iso build/os.fat12 build/os.zip.adjusted \ printf "\r\n:: %02x" $$i | dd bs=1 seek=$$(( ${bytes_gpt_mirror_start} + ( ($$i) - 1) * 128 + 16)) count=7 conv=notrunc of=$@; \ done # splice in zip at the end - set -x; dd skip=${bytes_zip_start} seek=${bytes_zip_start} bs=1 conv=notrunc if=build/os.zip.adjusted of=$@ + set -x; dd skip=${bytes_zip_start} seek=${bytes_zip_start} bs=1 conv=notrunc if=${bld}/os.zip.adjusted of=$@ chmod a+x-w $@ -build/os.file: ${os_filename} build/check_makefile +${bld}/os.file: ${os_filename} ${bld}/check_makefile file -kr $< > $@ -build/os.gdisk: ${os_filename} build/check_makefile +${bld}/os.gdisk: ${os_filename} ${bld}/check_makefile # gdisk commands: # * eXpert mode # * Print partition table @@ -351,41 +370,41 @@ build/os.gdisk: ${os_filename} build/check_makefile # * Quit printf '2\nx\np\ni\nr\no\nq\n' | ${gdisk_pipe_commands_slowly} | gdisk $< | tee $@ -build/os.offsets.hex: ${offset_names:%=build/offsets/%.hex} build/check_makefile - grep '^' ${offset_names:%=build/offsets/%.hex} | sed -e 's/:/: 0x/' | column -t > $@ +${bld}/os.offsets.hex: ${offset_names:%=${bld}/offsets/%.hex} ${bld}/check_makefile + grep '^' ${offset_names:%=${bld}/offsets/%.hex} | sed -e 's/:/: 0x/' | column -t > $@ -build/os.offsets.dec: ${offset_names:%=build/offsets/%.dec} build/check_makefile - grep '^' ${offset_names:%=build/offsets/%.dec} | sed -e 's/:/: /' | column -t > $@ +${bld}/os.offsets.dec: ${offset_names:%=${bld}/offsets/%.dec} ${bld}/check_makefile + grep '^' ${offset_names:%=${bld}/offsets/%.dec} | sed -e 's/:/: /' | column -t > $@ -build/offsets/%.hex: build/offsets/%.dec +${bld}/offsets/%.hex: ${bld}/offsets/%.dec printf '%x\n' $$(cat $<) > $@ -build/os.hex_with_offsets: ${os_filename} build/os.offsets.hex +${bld}/os.hex_with_offsets: ${os_filename} ${bld}/os.offsets.hex hexdump -C $< \ - | grep -E -e "($$(cat build/os.offsets.hex | cut -d '=' -f 2 | sed -e 's/^[[:space:]]*0x\(.*\).$$/^\10/' | tr '\n' '|')^)" --color=yes > $@ + | grep -E -e "($$(cat ${bld}/os.offsets.hex | cut -d '=' -f 2 | sed -e 's/^[[:space:]]*0x\(.*\).$$/^\10/' | tr '\n' '|')^)" --color=yes > $@ -build/os.ndisasm.disasm: ${os_filename} utils/compact-ndisasm.sh build/check_makefile +${bld}/os.ndisasm.disasm: ${os_filename} utils/compact-ndisasm.sh ${bld}/check_makefile ./utils/compact-ndisasm.sh $< $@ -build/os.reasm.asm: build/os.ndisasm.disasm build/check_makefile +${bld}/os.reasm.asm: ${bld}/os.ndisasm.disasm ${bld}/check_makefile sed -e 's/^[^ ]\+ \+[^ ]\+ \+//' $< > $@ -build/test_pass/noemu_%.reasm build/%.reasm: build/%.reasm.asm ${os_filename} utils/compact-ndisasm.sh build/check_makefile +${bld}/test_pass/noemu_%.reasm ${bld}/%.reasm: ${bld}/%.reasm.asm ${os_filename} utils/compact-ndisasm.sh ${bld}/check_makefile # For now ignore this test, since we cannot have a reliable re-assembly of arbitrary data. - touch build/test_pass/noemu_$*.reasm build/$*.reasm + touch ${bld}/test_pass/noemu_$*.reasm ${bld}/$*.reasm # nasm $< -o $@ # @echo "diff $@ ${os_filename}" # @diff $@ ${os_filename} \ # && echo "Re-assembled file is identical to ${os_filename}" \ -# || (./utils/compact-ndisasm.sh $@ build/os.reasm.disasm; \ -# echo "Re-assembled file is different from ${os_filename}. Use meld build/os.ndisasm.disasm build/os.reasm.disasm to see differences."; \ +# || (./utils/compact-ndisasm.sh $@ ${bld}/os.reasm.disasm; \ +# echo "Re-assembled file is different from ${os_filename}. Use meld ${bld}/os.ndisasm.disasm ${bld}/os.reasm.disasm to see differences."; \ # exit 0) -#os.arm.disasm: ${os_filename} build/check_makefile +#os.arm.disasm: ${os_filename} ${bld}/check_makefile # arm-none-eabi-objdump --endian=little -marm -b binary -D --adjust-vma=0x8000 $< > $@ .PHONY: clean -clean: build/check_makefile +clean: ${bld}/check_makefile rm -f ${built_files} ${temp_files} for d in $$(echo ${more_built_directories} ${temp_directories} | tr ' ' '\n' | sort --reverse); do \ if test -e "$$d"; then \ @@ -393,67 +412,67 @@ clean: build/check_makefile fi; \ done -.gitignore: build/check_makefile +.gitignore: ${bld}/check_makefile for f in ${built_files}; do echo "/$$f"; done | sort > $@ .PHONY: test -test: ${tests_emu:test/%=build/test_pass/emu_%} \ - ${tests_noemu:test/%=build/test_pass/noemu_%} \ - ${tests_requiring_sudo:test/%=build/test_pass/sudo_%} \ +test: ${tests_emu:test/%=${bld}/test_pass/emu_%} \ + ${tests_noemu:test/%=${bld}/test_pass/noemu_%} \ + ${tests_requiring_sudo:test/%=${bld}/test_pass/sudo_%} \ all \ - build/check_makefile + ${bld}/check_makefile .PHONY: ${tests_emu} -${tests_emu}: build/test_pass/emu_$$(@F) +${tests_emu}: ${bld}/test_pass/emu_$$(@F) -build/test_pass/emu_% deploy-screenshots/%.png deploy-screenshots/%-anim.gif: \ +${bld}/test_pass/emu_% deploy-screenshots/%.png deploy-screenshots/%-anim.gif: \ ${os_filename} \ - build/checkerboard_800x600.xbm \ + ${bld}/checkerboard_800x600.xbm \ utils/gui-wrapper.sh utils/ansi-screenshots/ansi_screenshot.sh utils/ansi-screenshots/to_ansi.sh \ test/%.sh \ - build/check_makefile \ - | build/test_pass deploy-screenshots + ${bld}/check_makefile \ + | ${bld}/test_pass deploy-screenshots ./utils/gui-wrapper.sh 800x600x24 ./test/$*.sh $< - touch build/test_pass/emu_$* + touch ${bld}/test_pass/emu_$* .PHONY: test/noemu test/noemu: ${tests_noemu:test/%=build/test_pass/noemu_%} build/check_makefile -build/test_pass/noemu_zip: ${os_filename} build/check_makefile +${bld}/test_pass/noemu_zip: ${os_filename} ${bld}/check_makefile unzip -t ${os_filename} touch $@ -build/test_pass/noemu_sizes: build/os.32k ${os_filename} build/check_makefile - test "$$(utils/file-length.sh -c build/os.32k)" = "$$((32*1024))" +${bld}/test_pass/noemu_sizes: ${bld}/os.32k ${os_filename} ${bld}/check_makefile + test "$$(utils/file-length.sh -c ${bld}/os.32k)" = "$$((32*1024))" test "$$(utils/file-length.sh -c ${os_filename})" = "$$((1440*1024))" touch $@ # check that the fat filesystem has the correct contents -build/test_pass/noemu_fat12_contents: ${os_filename} ${dep_bytes_fat12_start} build/check_makefile +${bld}/test_pass/noemu_fat12_contents: ${os_filename} ${dep_bytes_fat12_start} ${bld}/check_makefile mdir -i "$<@@${bytes_fat12_start}" :: | grep -E "^os[[:space:]]+zip[[:space:]]+" touch $@ .PHONY: test/requiring_sudo -test/requiring_sudo: ${tests_requiring_sudo:test/%=build/test_pass/sudo_%} build/check_makefile +test/requiring_sudo: ${tests_requiring_sudo:test/%=${bld}/test_pass/sudo_%} ${bld}/check_makefile # check that the fat filesystem can be mounted and has the correct contents -build/test_pass/sudo_fat12_mount: ${os_filename} ${dep_bytes_fat12_start} build/check_makefile | build/mnt_fat12 - sudo umount build/mnt_fat12 || true - sudo mount -o loop,ro,offset=${bytes_fat12_start} $< build/mnt_fat12 - ls -l build/mnt_fat12 | grep os.zip - sudo umount build/mnt_fat12 +${bld}/test_pass/sudo_fat12_mount: ${os_filename} ${dep_bytes_fat12_start} ${bld}/check_makefile | ${bld}/mnt_fat12 + sudo umount ${bld}/mnt_fat12 || true + sudo mount -o loop,ro,offset=${bytes_fat12_start} $< ${bld}/mnt_fat12 + ls -l ${bld}/mnt_fat12 | grep os.zip + sudo umount ${bld}/mnt_fat12 touch $@ -build/test_pass/sudo_iso_mount: ${os_filename} build/check_makefile | build/mnt_iso - sudo umount build/mnt_iso || true - grep '^' build/offsets/* # debug failure to mount the ISO9660 filesystem - (sudo mount -o loop,ro $< build/mnt_iso) || true +${bld}/test_pass/sudo_iso_mount: ${os_filename} ${bld}/check_makefile | ${bld}/mnt_iso + sudo umount ${bld}/mnt_iso || true + grep '^' ${bld}/offsets/* # debug failure to mount the ISO9660 filesystem + (sudo mount -o loop,ro $< ${bld}/mnt_iso) || true dmesg | tail # debug failure to mount the ISO9660 filesystem hexdump -C ${os_filename} - ls -l build/mnt_iso | grep os.zip - sudo umount build/mnt_iso - sudo mount -o loop,ro $< build/mnt_iso - sudo umount build/mnt_iso + ls -l ${bld}/mnt_iso | grep os.zip + sudo umount ${bld}/mnt_iso + sudo mount -o loop,ro $< ${bld}/mnt_iso + sudo umount ${bld}/mnt_iso touch $@ .PHONY: test/macos @@ -471,8 +490,8 @@ test/macos-sh-x11: screencapture deploy-screenshots/macos-sh-x11-screencapture.png .PHONY: test/macos-sh -test/macos-sh: build/check_makefile \ - build/checkerboard_1024x768.png \ +test/macos-sh: ${bld}/check_makefile \ + ${bld}/checkerboard_1024x768.png \ | deploy-screenshots osascript -e 'tell app "Terminal" to do script "'"$$PWD"'/${os_filename}"' sleep 2 @@ -485,18 +504,18 @@ test/macos-sh: build/check_makefile \ # See https://wiki.osdev.org/EFI#Emulation to emulate an UEFI system with qemu, to test the EFI boot from hdd / cd / fd (?). # Create checkerboard background -build/checkerboard_%.png: build/check_makefile +${bld}/checkerboard_%.png: ${bld}/check_makefile convert -size "$*" \ tile:pattern:checkerboard \ -auto-level +level-colors 'gray(192),gray(128)' \ $@ -build/checkerboard_%.xbm: build/check_makefile +${bld}/checkerboard_%.xbm: ${bld}/check_makefile convert -size "$*" \ tile:pattern:checkerboard \ -auto-level \ $@ # Temporary files -build/bochsrc build/bochscontinue build/twm_cfg build/virtualbox.img: +${bld}/bochsrc ${bld}/bochscontinue ${bld}/twm_cfg ${bld}/virtualbox.img: touch $@ -- 2.34.1 From 7dcdca9649e1710dbc455fbe181eba9e20d3f010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Sun, 26 Aug 2018 20:39:37 +0200 Subject: [PATCH 04/16] Check that the build is reproducible --- .gitignore | 1 + .travis.yml | 2 ++ Makefile | 73 ++++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 55 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index a93bb71..4eb63e1 100644 --- a/.gitignore +++ b/.gitignore @@ -96,6 +96,7 @@ /build/test_pass/emu_virtualbox /build/test_pass/noemu_fat12_contents /build/test_pass/noemu_os.reasm +/build/test_pass/noemu_reproducible_build /build/test_pass/noemu_sizes /build/test_pass/noemu_zip /build/test_pass/sudo_fat12_mount diff --git a/.travis.yml b/.travis.yml index 6f4e54c..c83ad48 100644 --- a/.travis.yml +++ b/.travis.yml @@ -61,6 +61,7 @@ matrix: - brew install mtools - brew cask install gdisk - brew install imagemagick # --with-x11 + - brew install libfaketime script: make all test/noemu test/macos - os: linux env: MODE=qemu-system-i386-floppy @@ -120,6 +121,7 @@ install: - import --version || test "$MODE" = osx-test # import --version has a non-zero exit code on the Travis installation - Xvfb -help - ssh -V + - faketime --version - ./utils/install-sikulix.sh - export PATH="$HOME/sikulix/:$PATH" - if test "$MODE" = virtualbox -o "$MODE" = self-test; then ./utils/install-virtualbox.sh; fi diff --git a/Makefile b/Makefile index 5870d21..fd93411 100644 --- a/Makefile +++ b/Makefile @@ -2,14 +2,15 @@ MAKEFLAGS = --warn-undefined-variables SHELL = bash -euET -o pipefail -c .SECONDEXPANSION: -os_filename = os.bat -BUILD_DIR = build -bld = ${BUILD_DIR} +OS_FILENAME = os.bat +os_filename = ${OS_FILENAME} +BUILD_DIR = build +bld = ${BUILD_DIR} tests_emu = test/qemu-system-i386-floppy test/qemu-system-i386-cdrom test/qemu-system-arm test/virtualbox test/bochs test/gui-sh test/dosbox tests_requiring_sudo = test/fat12_mount test/iso_mount -tests_noemu = test/zip test/os.reasm test/sizes test/fat12_contents +tests_noemu = test/zip test/os.reasm test/sizes test/fat12_contents test/reproducible_build -commit_timestamp = "$$(git log -1 --pretty=format:%ad --date=iso8601-strict)" +commit_timestamp = "$$(git log -1 --pretty=format:%ad --date=format:"%Y%m%d%H%m.%S")" commit_faketime = "$$(git log -1 --pretty=format:%ad --date=format:"%Y-%m-%d %H:%m:%S")" offset_names = bytes_os_size \ @@ -47,6 +48,8 @@ more_offset_names = ${offset_names} \ more_offset_dec = ${more_offset_names:%=${bld}/offsets/%.dec} more_offset_hex = ${more_offset_names:%=${bld}/offsets/%.hex} +reproducible_os_filename="${bld}/reproducible_$$(basename "${os_filename}")" + # + os.arm.disasm # + os.reasm.disasm built_files = ${os_filename} \ @@ -134,7 +137,7 @@ ${bld}/makefile_w_arnings: | $${@D} ${built_files}: | $${@D} ${bld}/makefile_w_arnings: Makefile - @unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; make -n --warn-undefined-variables BUILD_DIR=${BUILD_DIR} test 2>$@ 1>/dev/null || make -n --warn-undefined-variables test + @unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; make -n --warn-undefined-variables OS_FILENAME=${OS_FILENAME} BUILD_DIR=${BUILD_DIR} test 2>$@ 1>/dev/null || make -n --warn-undefined-variables test # Check that the file ${bld}/makefile_w_arnings is present, and that it does not contain the string "warn". ${bld}/check_makefile_w_arnings: ${bld}/makefile_w_arnings @@ -148,7 +151,7 @@ ${bld}/check_makefile: ${bld}/check_makefile_w_arnings ${bld}/check_makefile_tar @touch $@ ${bld}/makefile_database: Makefile ${bld}/check_makefile_w_arnings - @unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; make -rpn BUILD_DIR=${BUILD_DIR} | sed -n -e '/^# Make data base,/,$$p' > $@ + @unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; make -rpn OS_FILENAME=${OS_FILENAME} BUILD_DIR=${BUILD_DIR} | sed -n -e '/^# Make data base,/,$$p' > $@ ${bld}/makefile_database_files: ${bld}/makefile_database ${bld}/check_makefile_w_arnings @sed -n -e '/^# Files$$/,/^# files hash-table stats:$$/p' $< > $@ @@ -181,7 +184,7 @@ ${bld}/os.32k: example-os/os.asm ${bld}/check_makefile ${bld}/os.iso: ${bld}/iso_files/os.zip ${bld}/iso_files/boot/iso_boot.sys ${bld}/check_makefile cp -a -- ${bld}/iso_files ${bld}/iso_files.tmp - find ${bld}/iso_files.tmp -depth -exec touch -d ${commit_timestamp} '{}' ';' + find ${bld}/iso_files.tmp -depth -exec touch -t ${commit_timestamp} '{}' ';' faketime -f ${commit_faketime} mkisofs \ --input-charset utf-8 \ -rock \ @@ -220,12 +223,12 @@ ${eval ${call offset,sectors_os_size, $${bytes_os_size} / $${sector_size}, ${eval ${call offset,tracks_os_size, $${sectors_os_size} / $${os_floppy_chs_s}, sectors_os_size,}} # round up -${eval ${call offset,bytes_iso_size, $$$$(utils/file-length.sh -c ${bld}/os.iso), ,${bld}/os.iso}} +${eval ${call offset,bytes_iso_size, $$$$(utils/file-length.sh -c ${bld}/os.iso), ,${bld}/os.iso}} ${eval ${call offset,sectors_iso_size, ${call div_round_up,$${bytes_iso_size},$${sector_size}}, bytes_iso_size,}} ${eval ${call offset,tracks_iso_size, ${call div_round_up,$${sectors_iso_size},$${os_floppy_chs_s}}, sectors_iso_size,}} # round up -${eval ${call offset,bytes_zip_size, $$$$(utils/file-length.sh -c ${bld}/os.zip), ,${bld}/os.zip}} +${eval ${call offset,bytes_zip_size, $$$$(utils/file-length.sh -c ${bld}/os.zip), ,${bld}/os.zip}} ${eval ${call offset,sectors_zip_size, ${call div_round_up,$${bytes_zip_size},$${sector_size}}, bytes_zip_size,}} ${eval ${call offset,tracks_zip_size, ${call div_round_up,$${sectors_zip_size},$${os_floppy_chs_s}}, sectors_zip_size,}} @@ -285,7 +288,7 @@ ${bld}/os.zip: ${bld}/os.32k ${bld}/check_makefile # builds. mkdir -p ${bld}/os.32k.tmp cp -a $< ${bld}/os.32k.tmp/os.32k - touch -d ${commit_timestamp} ${bld}/os.32k.tmp/os.32k + touch -t ${commit_timestamp} ${bld}/os.32k.tmp/os.32k (cd ${bld}/os.32k.tmp/ && zip -X ../os.zip os.32k) rm ${bld}/os.32k.tmp/os.32k rmdir ${bld}/os.32k.tmp @@ -298,10 +301,10 @@ ${bld}/os.zip.adjusted: ${bld}/os.zip ${dep_bytes_zip_start} ${bld}/check_makefi gdisk_pipe_commands_slowly=while read str; do echo "$$str"; printf "\033[1;33m%s\033[m\n" "$$str" >&2; sleep 0.01; done -commit_hash_as_guid=$(git log -1 --pretty=format:%H | sed -e 's/^\(.\{8\}\)\(.\{4\}\)\(.\{4\}\)\(.\{4\}\)\(.\{12\}\).*$$/\1-\2-\3-\4-\5/' | tr '[:lower:]' '[:upper:]') +commit_hash_as_guid=$$(git log -1 --pretty=format:%H | sed -e 's/^\(.\{8\}\)\(.\{4\}\)\(.\{4\}\)\(.\{4\}\)\(.\{12\}\).*$$/\1-\2-\3-\4-\5/' | tr '[:lower:]' '[:upper:]') git_dirty=test -n "$$(git diff --shortstat)" -gpt_disk_guid=${commit_hash_as_guid}$$(if $git_dirty; then printf '0'; else printf '2'; fi) -gpt_partition_guid=${commit_hash_as_guid}$$(if $git_dirty; then printf '1'; else printf '3'; fi) +gpt_disk_guid=${commit_hash_as_guid}$$(if $$git_dirty; then printf '0'; else printf '2'; fi) +gpt_partition_guid=${commit_hash_as_guid}$$(if $$git_dirty; then printf '1'; else printf '3'; fi) ${os_filename}: ${bld}/os.32k ${bld}/os.iso ${bld}/os.fat12 ${bld}/os.zip.adjusted \ ${dep_bytes_header_32k_start} \ @@ -405,7 +408,11 @@ ${bld}/test_pass/noemu_%.reasm ${bld}/%.reasm: ${bld}/%.reasm.asm ${os_filename} .PHONY: clean clean: ${bld}/check_makefile - rm -f ${built_files} ${temp_files} + rm -f ${built_files} ${temp_files} ${bld}/reproducible_${os_filename} + if test -d ${bld}/reproducible; then \ + unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \ + make OS_FILENAME=${reproducible_os_filename} BUILD_DIR=${bld}/reproducible clean; \ + fi for d in $$(echo ${more_built_directories} ${temp_directories} | tr ' ' '\n' | sort --reverse); do \ if test -e "$$d"; then \ rmdir "$$d"; \ @@ -422,8 +429,15 @@ test: ${tests_emu:test/%=${bld}/test_pass/emu_%} \ all \ ${bld}/check_makefile -.PHONY: ${tests_emu} -${tests_emu}: ${bld}/test_pass/emu_$$(@F) +.PHONY: test/emu test/noemu test/requiring_sudo +test/emu: ${tests_emu} ${bld}/check_makefile +test/requiring_sudo: ${tests_requiring_sudo} ${bld}/check_makefile +test/noemu: ${tests_noemu} ${bld}/check_makefile + +.PHONY: ${tests_emu} ${tests_noemu} ${tests_requiring_sudo} +${tests_emu}: ${bld}/test_pass/emu_$$(@F) ${bld}/check_makefile +${tests_noemu}: ${bld}/test_pass/noemu_$$(@F) ${bld}/check_makefile +${tests_requiring_sudo}: ${bld}/test_pass/sudo_$$(@F) ${bld}/check_makefile ${bld}/test_pass/emu_% deploy-screenshots/%.png deploy-screenshots/%-anim.gif: \ ${os_filename} \ @@ -435,9 +449,6 @@ ${bld}/test_pass/emu_% deploy-screenshots/%.png deploy-screenshots/%-anim.gif: \ ./utils/gui-wrapper.sh 800x600x24 ./test/$*.sh $< touch ${bld}/test_pass/emu_$* -.PHONY: test/noemu -test/noemu: ${tests_noemu:test/%=build/test_pass/noemu_%} build/check_makefile - ${bld}/test_pass/noemu_zip: ${os_filename} ${bld}/check_makefile unzip -t ${os_filename} touch $@ @@ -517,5 +528,25 @@ ${bld}/checkerboard_%.xbm: ${bld}/check_makefile $@ # Temporary files -${bld}/bochsrc ${bld}/bochscontinue ${bld}/twm_cfg ${bld}/virtualbox.img: +${bld}/bochsrc ${bld}/bochscontinue ${bld}/twm_cfg ${bld}/virtualbox.img: ${bld}/check_makefile + touch $@ + +# DEBUG: ${bld}/os.hex_with_offsets ${bld}/os.offsets.hex +${bld}/test_pass/noemu_reproducible_build: ${os_filename} ${bld}/os.hex_with_offsets ${bld}/check_makefile +# Let some time pass so that any timestamp that may affect the result changes. + sleep 5 +# TODO: try to see if we can re-enable some of these variables without +# causing problems on macos. + unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \ + make OS_FILENAME=${reproducible_os_filename} BUILD_DIR=${bld}/reproducible clean + unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \ + make OS_FILENAME=${reproducible_os_filename} BUILD_DIR=${bld}/reproducible ${reproducible_os_filename} \ + ${bld}/reproducible/os.hex_with_offsets +# Check that the second build produced the same file. + if ! diff ${os_filename} ${reproducible_os_filename}; then \ + diff ${bld}/os.hex_with_offsets ${bld}/reproducible/os.hex_with_offsets || true; \ + exit 1; \ + fi + unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \ + make OS_FILENAME=${reproducible_os_filename} BUILD_DIR=${bld}/reproducible clean touch $@ -- 2.34.1 From 01ea5827b28f9b26c7f167d05450ebe77ea3940a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Sun, 23 Sep 2018 09:37:04 +0200 Subject: [PATCH 05/16] Added COMMIT_TIMESTAMP_ISO_8601 as a Makefile parameter --- Makefile | 13 +++++++++---- appveyor.yml | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index fd93411..55e8177 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,22 @@ +#################################### +# Makefile configuration variables # +OS_FILENAME = os.bat +BUILD_DIR = build +COMMIT_TIMESTAMP_ISO_8601 = $$(git log -1 --pretty=format:%ad --date=iso8601-strict) +#################################### + MAKEFLAGS = --warn-undefined-variables SHELL = bash -euET -o pipefail -c .SECONDEXPANSION: -OS_FILENAME = os.bat os_filename = ${OS_FILENAME} -BUILD_DIR = build bld = ${BUILD_DIR} tests_emu = test/qemu-system-i386-floppy test/qemu-system-i386-cdrom test/qemu-system-arm test/virtualbox test/bochs test/gui-sh test/dosbox tests_requiring_sudo = test/fat12_mount test/iso_mount tests_noemu = test/zip test/os.reasm test/sizes test/fat12_contents test/reproducible_build -commit_timestamp = "$$(git log -1 --pretty=format:%ad --date=format:"%Y%m%d%H%m.%S")" -commit_faketime = "$$(git log -1 --pretty=format:%ad --date=format:"%Y-%m-%d %H:%m:%S")" +commit_timestamp = "$$(date -d "${COMMIT_TIMESTAMP_ISO_8601}" '+%Y%m%d%H%m.%S')" +commit_faketime = "$$(date -d "${COMMIT_TIMESTAMP_ISO_8601}" '+%Y-%m-%d %H:%m:%S')" offset_names = bytes_os_size \ bytes_mbr_start \ diff --git a/appveyor.yml b/appveyor.yml index 786a59a..5aa9f8f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -57,7 +57,7 @@ install: - cmd: echo TODO install gnu make and dependencies build_script: - - cmd: make all + - cmd: make COMMIT_TIMESTAMP_ISO_8601="%APPVEYOR_REPO_COMMIT_TIMESTAMP%" all test_script: - cmd: cmd /C os.bat exit -- 2.34.1 From 94deee14b017f29d352f4b978979c79340e28f43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Sun, 23 Sep 2018 09:38:50 +0200 Subject: [PATCH 06/16] Avoid copying iso_files as a subfolder of an existing iso_files.tmp if the latter was not cleaned correctly --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 55e8177..949a8f0 100644 --- a/Makefile +++ b/Makefile @@ -188,7 +188,8 @@ ${bld}/os.32k: example-os/os.asm ${bld}/check_makefile nasm -w+macro-params -w+macro-selfref -w+orphan-labels -w+gnu-elf-extensions -o $@ $< ${bld}/os.iso: ${bld}/iso_files/os.zip ${bld}/iso_files/boot/iso_boot.sys ${bld}/check_makefile - cp -a -- ${bld}/iso_files ${bld}/iso_files.tmp + ! test -d ${bld}/iso_files.tmp + cp -a -T -- ${bld}/iso_files ${bld}/iso_files.tmp find ${bld}/iso_files.tmp -depth -exec touch -t ${commit_timestamp} '{}' ';' faketime -f ${commit_faketime} mkisofs \ --input-charset utf-8 \ -- 2.34.1 From 634a4a0ff388fdf47cc51ae2fc8fdb7bf8d1212d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Sun, 23 Sep 2018 09:40:42 +0200 Subject: [PATCH 07/16] Omit the build directory path from os.offsets.hex The build directory is not the same for the normal build and the build made to check for reproducibility. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 949a8f0..77b3d10 100644 --- a/Makefile +++ b/Makefile @@ -380,10 +380,10 @@ ${bld}/os.gdisk: ${os_filename} ${bld}/check_makefile printf '2\nx\np\ni\nr\no\nq\n' | ${gdisk_pipe_commands_slowly} | gdisk $< | tee $@ ${bld}/os.offsets.hex: ${offset_names:%=${bld}/offsets/%.hex} ${bld}/check_makefile - grep '^' ${offset_names:%=${bld}/offsets/%.hex} | sed -e 's/:/: 0x/' | column -t > $@ + grep '^' ${offset_names:%=${bld}/offsets/%.hex} | sed -e 's|^.*/||' -e 's/:/: 0x/' | column -t > $@ ${bld}/os.offsets.dec: ${offset_names:%=${bld}/offsets/%.dec} ${bld}/check_makefile - grep '^' ${offset_names:%=${bld}/offsets/%.dec} | sed -e 's/:/: /' | column -t > $@ + grep '^' ${offset_names:%=${bld}/offsets/%.dec} | sed -e 's|^.*/||' -e 's/:/: /' | column -t > $@ ${bld}/offsets/%.hex: ${bld}/offsets/%.dec printf '%x\n' $$(cat $<) > $@ -- 2.34.1 From 00bfba9251a9e1fd71d041e79d9458f733148091 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Mon, 27 Aug 2018 08:42:50 +0200 Subject: [PATCH 08/16] macos: mkisofs includes the path to its input as extra unwanted directories in the iso --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 862974d..3ac6d2d 100644 --- a/Makefile +++ b/Makefile @@ -206,7 +206,7 @@ ${bld}/os.iso: ${bld}/iso_files/os.zip ${bld}/iso_files/boot/iso_boot.sys ${bld} ! test -d ${bld}/iso_files.tmp cp -a -T -- ${bld}/iso_files ${bld}/iso_files.tmp find ${bld}/iso_files.tmp -depth -exec touch -t ${commit_timestamp} '{}' ';' - faketime -f ${commit_faketime} mkisofs \ + (cd ./${bld}/iso_files.tmp/ && faketime -f ${commit_faketime} mkisofs \ --input-charset utf-8 \ -rock \ -joliet \ @@ -215,8 +215,8 @@ ${bld}/os.iso: ${bld}/iso_files/os.zip ${bld}/iso_files/boot/iso_boot.sys ${bld} -no-emul-boot \ -boot-load-size 4 \ -pad \ - -output $@ \ - ./${bld}/iso_files.tmp/ + -output ../os.iso \ + .) rm -- ${bld}/iso_files.tmp/os.zip \ ${bld}/iso_files.tmp/boot/iso_boot.sys rmdir ${bld}/iso_files.tmp/boot/ -- 2.34.1 From 2e9ce6fc261b275b1be07084f005083e5576f993 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Sun, 23 Sep 2018 17:38:38 +0200 Subject: [PATCH 09/16] macos: date conversion needs different flags than in Linux --- Makefile | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3ac6d2d..302d46e 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,17 @@ tests_emu = test/qemu-system-i386-floppy test/qemu-system-i386-cdrom test/qemu-s tests_requiring_sudo = test/fat12_mount test/iso_mount tests_noemu = test/zip test/os.reasm test/sizes test/fat12_contents test/reproducible_build -commit_timestamp = "$$(date -d "${COMMIT_TIMESTAMP_ISO_8601}" '+%Y%m%d%H%m.%S')" -commit_faketime = "$$(date -d "${COMMIT_TIMESTAMP_ISO_8601}" '+%Y-%m-%d %H:%m:%S')" +# We truncate the timezone, because the Darwin version of date seems to lack +# the %:z format (for ±HH:MM timezone). +define date_command + if test "$$(uname -s)" = Darwin; then \ + date -j -f %Y-%m-%dT%H:%M:%S $$(echo ${1} | cut -c 1-19) ${2}; \ + else \ + date -d ${1} ${2}; \ + fi +endef +commit_timestamp = "$$(${call date_command,"${COMMIT_TIMESTAMP_ISO_8601}",'+%Y%m%d%H%m.%S'})" +commit_faketime = "$$(${call date_command,"${COMMIT_TIMESTAMP_ISO_8601}",'+%Y-%m-%d %H:%m:%S'})" offset_names = bytes_os_size \ bytes_mbr_start \ -- 2.34.1 From 08901554b0c166246a79f54c660832e1d74a36c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Wed, 29 Aug 2018 23:50:15 +0200 Subject: [PATCH 10/16] Screenshots directory is now an option, to avoid conflicts with the reproducible build's make clean --- Makefile | 67 +++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 77b3d10..862974d 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ # Makefile configuration variables # OS_FILENAME = os.bat BUILD_DIR = build +SCREENSHOTS_DIR = deploy-screenshots COMMIT_TIMESTAMP_ISO_8601 = $$(git log -1 --pretty=format:%ad --date=iso8601-strict) #################################### @@ -11,6 +12,7 @@ SHELL = bash -euET -o pipefail -c os_filename = ${OS_FILENAME} bld = ${BUILD_DIR} +screenshots = ${SCREENSHOTS_DIR} tests_emu = test/qemu-system-i386-floppy test/qemu-system-i386-cdrom test/qemu-system-arm test/virtualbox test/bochs test/gui-sh test/dosbox tests_requiring_sudo = test/fat12_mount test/iso_mount tests_noemu = test/zip test/os.reasm test/sizes test/fat12_contents test/reproducible_build @@ -53,7 +55,7 @@ more_offset_names = ${offset_names} \ more_offset_dec = ${more_offset_names:%=${bld}/offsets/%.dec} more_offset_hex = ${more_offset_names:%=${bld}/offsets/%.hex} -reproducible_os_filename="${bld}/reproducible_$$(basename "${os_filename}")" +reproducible_os_filename="${bld}/reproduced_$$(basename "${os_filename}")" # + os.arm.disasm # + os.reasm.disasm @@ -96,8 +98,8 @@ built_files = ${os_filename} \ ${tests_emu:test/%=${bld}/test_pass/emu_%} \ ${tests_noemu:test/%=${bld}/test_pass/noemu_%} \ ${tests_requiring_sudo:test/%=${bld}/test_pass/sudo_%} \ - ${tests_emu:test/%=deploy-screenshots/%.png} \ - ${tests_emu:test/%=deploy-screenshots/%-anim.gif} + ${tests_emu:test/%=${screenshots}/%.png} \ + ${tests_emu:test/%=${screenshots}/%-anim.gif} # Temporary copies used to adjust timestamps for reproducible builds. # These are normally created and deleted within a single target, but @@ -113,7 +115,7 @@ built_directories = ${bld}/iso_files/boot \ ${bld}/mnt_fat12 \ ${bld}/mnt_iso \ ${bld}/test_pass \ - deploy-screenshots + ${screenshots} more_built_directories = ${built_directories} ${bld} os_image_size_kb = 1440 @@ -142,7 +144,14 @@ ${bld}/makefile_w_arnings: | $${@D} ${built_files}: | $${@D} ${bld}/makefile_w_arnings: Makefile - @unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; make -n --warn-undefined-variables OS_FILENAME=${OS_FILENAME} BUILD_DIR=${BUILD_DIR} test 2>$@ 1>/dev/null || make -n --warn-undefined-variables test + @unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \ + make -n --warn-undefined-variables \ + OS_FILENAME=${OS_FILENAME} \ + BUILD_DIR=${BUILD_DIR} \ + SCREENSHOTS_DIR=${SCREENSHOTS_DIR} \ + COMMIT_TIMESTAMP_ISO_8601=${COMMIT_TIMESTAMP_ISO_8601} \ + test 2>$@ 1>/dev/null \ + || cat $@ # Check that the file ${bld}/makefile_w_arnings is present, and that it does not contain the string "warn". ${bld}/check_makefile_w_arnings: ${bld}/makefile_w_arnings @@ -156,7 +165,13 @@ ${bld}/check_makefile: ${bld}/check_makefile_w_arnings ${bld}/check_makefile_tar @touch $@ ${bld}/makefile_database: Makefile ${bld}/check_makefile_w_arnings - @unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; make -rpn OS_FILENAME=${OS_FILENAME} BUILD_DIR=${BUILD_DIR} | sed -n -e '/^# Make data base,/,$$p' > $@ + @unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \ + make -rpn \ + OS_FILENAME=${OS_FILENAME} \ + BUILD_DIR=${BUILD_DIR} \ + SCREENSHOTS_DIR=${SCREENSHOTS_DIR} \ + COMMIT_TIMESTAMP_ISO_8601=${COMMIT_TIMESTAMP_ISO_8601} \ + | sed -n -e '/^# Make data base,/,$$p' > $@ ${bld}/makefile_database_files: ${bld}/makefile_database ${bld}/check_makefile_w_arnings @sed -n -e '/^# Files$$/,/^# files hash-table stats:$$/p' $< > $@ @@ -414,10 +429,14 @@ ${bld}/test_pass/noemu_%.reasm ${bld}/%.reasm: ${bld}/%.reasm.asm ${os_filename} .PHONY: clean clean: ${bld}/check_makefile - rm -f ${built_files} ${temp_files} ${bld}/reproducible_${os_filename} + rm -f ${built_files} ${temp_files} ${bld}/${reproducible_os_filename} if test -d ${bld}/reproducible; then \ unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \ - make OS_FILENAME=${reproducible_os_filename} BUILD_DIR=${bld}/reproducible clean; \ + make OS_FILENAME=${reproducible_os_filename} \ + BUILD_DIR=${bld}/reproducible \ + SCREENSHOTS_DIR=${bld}/reproducible/screenshots \ + COMMIT_TIMESTAMP_ISO_8601=${COMMIT_TIMESTAMP_ISO_8601} \ + clean; \ fi for d in $$(echo ${more_built_directories} ${temp_directories} | tr ' ' '\n' | sort --reverse); do \ if test -e "$$d"; then \ @@ -445,13 +464,13 @@ ${tests_emu}: ${bld}/test_pass/emu_$$(@F) ${bld}/check_makefile ${tests_noemu}: ${bld}/test_pass/noemu_$$(@F) ${bld}/check_makefile ${tests_requiring_sudo}: ${bld}/test_pass/sudo_$$(@F) ${bld}/check_makefile -${bld}/test_pass/emu_% deploy-screenshots/%.png deploy-screenshots/%-anim.gif: \ +${bld}/test_pass/emu_% ${screenshots}/%.png ${screenshots}/%-anim.gif: \ ${os_filename} \ ${bld}/checkerboard_800x600.xbm \ utils/gui-wrapper.sh utils/ansi-screenshots/ansi_screenshot.sh utils/ansi-screenshots/to_ansi.sh \ test/%.sh \ ${bld}/check_makefile \ - | ${bld}/test_pass deploy-screenshots + | ${bld}/test_pass ${screenshots} ./utils/gui-wrapper.sh 800x600x24 ./test/$*.sh $< touch ${bld}/test_pass/emu_$* @@ -503,19 +522,19 @@ test/macos-sh-x11: sleep 5; \ DISPLAY=:42 xterm -e ./${os_filename} & \ sleep 5; \ -# DISPLAY=:42 import -window root deploy-screenshots/macos-sh-x11.png - screencapture deploy-screenshots/macos-sh-x11-screencapture.png +# DISPLAY=:42 import -window root ${screenshots}/macos-sh-x11.png + screencapture ${screenshots}/macos-sh-x11-screencapture.png .PHONY: test/macos-sh test/macos-sh: ${bld}/check_makefile \ ${bld}/checkerboard_1024x768.png \ - | deploy-screenshots + | ${screenshots} osascript -e 'tell app "Terminal" to do script "'"$$PWD"'/${os_filename}"' sleep 2 osascript -e 'tell app "Terminal" to activate' sleep 5 (date +%n && sleep 0.2 && date +%n) || true - screencapture deploy-screenshots/screencapture-os-bat.png + screencapture ${screenshots}/screencapture-os-bat.png ./utils/gui-wrapper-mac.sh 1024x768x24 ./test/gui-sh-mac.sh ${os_filename} # See https://wiki.osdev.org/EFI#Emulation to emulate an UEFI system with qemu, to test the EFI boot from hdd / cd / fd (?). @@ -544,15 +563,27 @@ ${bld}/test_pass/noemu_reproducible_build: ${os_filename} ${bld}/os.hex_with_off # TODO: try to see if we can re-enable some of these variables without # causing problems on macos. unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \ - make OS_FILENAME=${reproducible_os_filename} BUILD_DIR=${bld}/reproducible clean + make OS_FILENAME=${reproducible_os_filename} \ + BUILD_DIR=${bld}/reproducible \ + SCREENSHOTS_DIR=${bld}/reproducible/screenshots \ + COMMIT_TIMESTAMP_ISO_8601=${COMMIT_TIMESTAMP_ISO_8601} \ + clean unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \ - make OS_FILENAME=${reproducible_os_filename} BUILD_DIR=${bld}/reproducible ${reproducible_os_filename} \ - ${bld}/reproducible/os.hex_with_offsets + make OS_FILENAME=${reproducible_os_filename} \ + BUILD_DIR=${bld}/reproducible \ + SCREENSHOTS_DIR=${bld}/reproducible/screenshots \ + COMMIT_TIMESTAMP_ISO_8601=${COMMIT_TIMESTAMP_ISO_8601} \ + ${reproducible_os_filename} \ + ${bld}/reproducible/os.hex_with_offsets # Check that the second build produced the same file. if ! diff ${os_filename} ${reproducible_os_filename}; then \ diff ${bld}/os.hex_with_offsets ${bld}/reproducible/os.hex_with_offsets || true; \ exit 1; \ fi unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \ - make OS_FILENAME=${reproducible_os_filename} BUILD_DIR=${bld}/reproducible clean + make OS_FILENAME=${reproducible_os_filename} \ + BUILD_DIR=${bld}/reproducible \ + SCREENSHOTS_DIR=${bld}/reproducible/screenshots \ + COMMIT_TIMESTAMP_ISO_8601=${COMMIT_TIMESTAMP_ISO_8601} \ + clean touch $@ -- 2.34.1 From 07e12d30628b2f94d15af7f5aea09ae05b347e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Fri, 31 Aug 2018 09:28:39 +0200 Subject: [PATCH 11/16] cp -T option does not exist on on macos --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 302d46e..9da4634 100644 --- a/Makefile +++ b/Makefile @@ -211,9 +211,10 @@ ${more_built_directories}: Makefile ${bld}/os.32k: example-os/os.asm ${bld}/check_makefile nasm -w+macro-params -w+macro-selfref -w+orphan-labels -w+gnu-elf-extensions -o $@ $< +cp_T_option = $$(if test "$$(uname -s)" = Darwin; then echo ''; else echo '-T'; fi) ${bld}/os.iso: ${bld}/iso_files/os.zip ${bld}/iso_files/boot/iso_boot.sys ${bld}/check_makefile ! test -d ${bld}/iso_files.tmp - cp -a -T -- ${bld}/iso_files ${bld}/iso_files.tmp + cp -a ${cp_T_option} -- ${bld}/iso_files ${bld}/iso_files.tmp find ${bld}/iso_files.tmp -depth -exec touch -t ${commit_timestamp} '{}' ';' (cd ./${bld}/iso_files.tmp/ && faketime -f ${commit_faketime} mkisofs \ --input-charset utf-8 \ -- 2.34.1 From 60a7f054b32eab79b0255e4eade0d0bf1a27cbae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Sun, 23 Sep 2018 09:41:55 +0200 Subject: [PATCH 12/16] appveyor: clone_depth is unnecessary if shallow_clone is used --- appveyor.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 5aa9f8f..aae49e8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,7 +1,5 @@ max_jobs: 1 -# Are both necessary? shallow_clone: true -clone_depth: 1 image: Visual Studio 2015 -- 2.34.1 From 6f5522bdcb8ae697719b419bbfead214d64ffa93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Fri, 31 Aug 2018 09:36:04 +0200 Subject: [PATCH 13/16] Installed datefudge on Appveyor via Cygwin --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index aae49e8..f685024 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -22,7 +22,7 @@ install: - cmd: cd .. - cmd: set PATH=%cd%\gdisk\;%PATH% # mkisofs,zip,ImageMagick - - cmd: C:\cygwin\setup-x86.exe --no-replaceonreboot --no-shortcuts --quiet-mode --root c:\cygwin --packages mkisofs,genisoimage,zip,ImageMagick,xorg-server-extra + - cmd: C:\cygwin\setup-x86.exe --no-replaceonreboot --no-shortcuts --quiet-mode --root c:\cygwin --packages mkisofs,genisoimage,zip,ImageMagick,xorg-server-extra,datefudge - cmd: C:\cygwin\bin\find C:\cygwin -iname "import*" - cmd: C:\cygwin\bin\find C:\cygwin -iname "convert*" # mtools -- 2.34.1 From 24fe32c3b51c69cc3509cd6a1cd4b2f8a4c01028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Sat, 22 Sep 2018 00:40:07 +0200 Subject: [PATCH 14/16] Circumvent the fact that faketime does not work on system binaries in macos --- .gitignore | 3 +++ Makefile | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 4eb63e1..d129e9f 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,6 @@ /deploy-screenshots/virtualbox-anim.gif /deploy-screenshots/virtualbox.png /os.bat +/utils/mcopy +/utils/mformat +/utils/mkisofs diff --git a/Makefile b/Makefile index 9da4634..708dbe8 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,8 @@ built_files = ${os_filename} \ ${tests_noemu:test/%=${bld}/test_pass/noemu_%} \ ${tests_requiring_sudo:test/%=${bld}/test_pass/sudo_%} \ ${tests_emu:test/%=${screenshots}/%.png} \ - ${tests_emu:test/%=${screenshots}/%-anim.gif} + ${tests_emu:test/%=${screenshots}/%-anim.gif} \ + utils/mformat utils/mcopy utils/mkisofs # Temporary copies used to adjust timestamps for reproducible builds. # These are normally created and deleted within a single target, but @@ -211,12 +212,17 @@ ${more_built_directories}: Makefile ${bld}/os.32k: example-os/os.asm ${bld}/check_makefile nasm -w+macro-params -w+macro-selfref -w+orphan-labels -w+gnu-elf-extensions -o $@ $< +# Circumvent the fact that faketime does not work on system binaries in macos +./utils/mkisofs ./utils/mformat ./utils/mcopy: Makefile # TODO: depend on the mkisofs binary + cp $$(which $$(basename $@)) $@ + chmod u+x $@ + cp_T_option = $$(if test "$$(uname -s)" = Darwin; then echo ''; else echo '-T'; fi) -${bld}/os.iso: ${bld}/iso_files/os.zip ${bld}/iso_files/boot/iso_boot.sys ${bld}/check_makefile +${bld}/os.iso: ${bld}/iso_files/os.zip ${bld}/iso_files/boot/iso_boot.sys ./utils/mkisofs ${bld}/check_makefile ! test -d ${bld}/iso_files.tmp cp -a ${cp_T_option} -- ${bld}/iso_files ${bld}/iso_files.tmp find ${bld}/iso_files.tmp -depth -exec touch -t ${commit_timestamp} '{}' ';' - (cd ./${bld}/iso_files.tmp/ && faketime -f ${commit_faketime} mkisofs \ + UTILS="$$PWD/utils" (cd ./${bld}/iso_files.tmp/ && faketime -f ${commit_faketime} $$UTILS/mkisofs \ --input-charset utf-8 \ -rock \ -joliet \ @@ -292,14 +298,15 @@ ${eval ${call offset,bytes_gpt_mirror_start, $${bytes_gpt_mirror_end} - $${bytes ${eval ${call offset,bytes_zip_end, $${bytes_os_size}, bytes_os_size,}} os_fat12_partition = "$@@@${bytes_fat12_start}" -${bld}/os.fat12: ${bld}/os.zip ${dep_bytes_fat12_size} ${dep_bytes_fat12_start} ${dep_sectors_os_size} ${bld}/check_makefile +${bld}/os.fat12: ${bld}/os.zip ${dep_bytes_fat12_size} ${dep_bytes_fat12_start} ${dep_sectors_os_size} \ + ./utils/mformat ./utils/mcopy ${bld}/check_makefile set -x; dd if=/dev/zero bs=${sector_size} count=${sectors_os_size} of=$@ - faketime -f ${commit_faketime} mformat -v "Example OS" \ + faketime -f ${commit_faketime} ./utils/mformat -v "Example OS" \ -T ${sectors_fat12_size} \ -h ${os_floppy_chs_h} \ -s ${os_floppy_chs_s} \ -i ${os_fat12_partition} - faketime -f ${commit_faketime} mcopy -i ${os_fat12_partition} $< "::os.zip" + faketime -f ${commit_faketime} ./utils/mcopy -i ${os_fat12_partition} $< "::os.zip" ${bld}/iso_files/os.zip: ${bld}/os.zip ${bld}/check_makefile # TODO: make it so that the various file formats are mutual quines: -- 2.34.1 From 76ef64b5be2231ef49bb8affca809e785f9d83be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Sat, 1 Sep 2018 00:16:40 +0200 Subject: [PATCH 15/16] Use faketime or datefudge if the former is not installed --- Makefile | 10 +++++----- utils/faketime.sh | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) create mode 100755 utils/faketime.sh diff --git a/Makefile b/Makefile index 708dbe8..c0e7cfe 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ define date_command fi endef commit_timestamp = "$$(${call date_command,"${COMMIT_TIMESTAMP_ISO_8601}",'+%Y%m%d%H%m.%S'})" -commit_faketime = "$$(${call date_command,"${COMMIT_TIMESTAMP_ISO_8601}",'+%Y-%m-%d %H:%m:%S'})" +commit_timestamp_iso_8601 = ${COMMIT_TIMESTAMP_ISO_8601} offset_names = bytes_os_size \ bytes_mbr_start \ @@ -222,7 +222,7 @@ ${bld}/os.iso: ${bld}/iso_files/os.zip ${bld}/iso_files/boot/iso_boot.sys ./util ! test -d ${bld}/iso_files.tmp cp -a ${cp_T_option} -- ${bld}/iso_files ${bld}/iso_files.tmp find ${bld}/iso_files.tmp -depth -exec touch -t ${commit_timestamp} '{}' ';' - UTILS="$$PWD/utils" (cd ./${bld}/iso_files.tmp/ && faketime -f ${commit_faketime} $$UTILS/mkisofs \ + UTILS="$$PWD/utils" ./utils/faketime.sh ${commit_timestamp_iso_8601} sh -c '(cd ./${bld}/iso_files.tmp/ && "$$UTILS/mkisofs" \ --input-charset utf-8 \ -rock \ -joliet \ @@ -232,7 +232,7 @@ ${bld}/os.iso: ${bld}/iso_files/os.zip ${bld}/iso_files/boot/iso_boot.sys ./util -boot-load-size 4 \ -pad \ -output ../os.iso \ - .) + .)' rm -- ${bld}/iso_files.tmp/os.zip \ ${bld}/iso_files.tmp/boot/iso_boot.sys rmdir ${bld}/iso_files.tmp/boot/ @@ -301,12 +301,12 @@ os_fat12_partition = "$@@@${bytes_fat12_start}" ${bld}/os.fat12: ${bld}/os.zip ${dep_bytes_fat12_size} ${dep_bytes_fat12_start} ${dep_sectors_os_size} \ ./utils/mformat ./utils/mcopy ${bld}/check_makefile set -x; dd if=/dev/zero bs=${sector_size} count=${sectors_os_size} of=$@ - faketime -f ${commit_faketime} ./utils/mformat -v "Example OS" \ + ./utils/faketime.sh ${commit_timestamp_iso_8601} ./utils/mformat -v "Example OS" \ -T ${sectors_fat12_size} \ -h ${os_floppy_chs_h} \ -s ${os_floppy_chs_s} \ -i ${os_fat12_partition} - faketime -f ${commit_faketime} ./utils/mcopy -i ${os_fat12_partition} $< "::os.zip" + ./utils/faketime.sh ${commit_timestamp_iso_8601} ./utils/mcopy -i ${os_fat12_partition} $< "::os.zip" ${bld}/iso_files/os.zip: ${bld}/os.zip ${bld}/check_makefile # TODO: make it so that the various file formats are mutual quines: diff --git a/utils/faketime.sh b/utils/faketime.sh new file mode 100755 index 0000000..d36740c --- /dev/null +++ b/utils/faketime.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -euET -o pipefail + +timestamp_iso_8601="$1" +shift + +date_command() { + # TODO: substring or case … in Darwin*) + if test "$(uname -s)" = Darwin; then + date -j -f '%Y-%m-%dT%H:%M:%S' "$(echo "${1}" | cut -c 1-19)" "${2}"; + else + date -d "${1}" "${2}"; + fi +} + +if which faketime >/dev/null; then + ( set -x; faketime -f "$(date_command "${timestamp_iso_8601}" '+%Y-%m-%d %H:%m:%S')" "$@"; ) +elif which datefudge >/dev/null; then + ( set -x; datefudge --static "${timestamp_iso_8601}" "$@"; ) +else + echo "ERROR: command faketime or datefudge not found. Please install either command." + exit 1 +fi -- 2.34.1 From 6b0082dd68a543d0051dbd5a3399d1cd3d4126d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georges=20Dup=C3=A9ron?= Date: Sun, 23 Sep 2018 19:43:00 +0200 Subject: [PATCH 16/16] Display warning: reproducible builds do not work on Windows or macos --- Makefile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c0e7cfe..d6df3d1 100644 --- a/Makefile +++ b/Makefile @@ -595,7 +595,17 @@ ${bld}/test_pass/noemu_reproducible_build: ${os_filename} ${bld}/os.hex_with_off # Check that the second build produced the same file. if ! diff ${os_filename} ${reproducible_os_filename}; then \ diff ${bld}/os.hex_with_offsets ${bld}/reproducible/os.hex_with_offsets || true; \ - exit 1; \ + if test "$$(uname -s)" = Darwin -o "$$(uname -o)" = "Cygwin"; then \ + for i in `seq 5`; do \ + printf '\033[1;31m########################################################\033[m'; \ + done; \ + echo "REPRODUCIBLE BUILDS ARE UNSUPPORTED ON MACOS AND WINDOWS"; \ + for i in `seq 5`; do \ + printf '\033[1;31m########################################################\033[m'; \ + done; \ + else \ + exit 1; \ + fi; \ fi unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \ make OS_FILENAME=${reproducible_os_filename} \ -- 2.34.1