Compare commits

...

27 Commits

Author SHA1 Message Date
Georges Dupéron
239f050a4b
Split the clean target into clean and clean_reproducible 2018-10-02 03:19:01 +02:00
Georges Dupéron
017b038ac4
Split the Makefile into three parts (build framework, os-specific build recipes, tests)
Closes #1
2018-10-02 03:19:00 +02:00
Georges Dupéron
4fb06d5865
Merge branch 'misc-cleanup-and-debug' into master 2018-10-02 03:18:08 +02:00
Georges Dupéron
3669ba800e
Tests should fail if the OS filename argument is missing. 2018-10-02 03:18:08 +02:00
Georges Dupéron
584a8541c0
Fixed dependency in Makefile 2018-10-02 03:18:08 +02:00
Georges Dupéron
0e59aa2cc2
make -j 10 sometimes fails to mount the resulting ISO9660 filesystem, added more information and a dump of the OS binary to help debug the issue 2018-10-02 03:18:08 +02:00
Georges Dupéron
f98acb7662
debug sikuli installation error 2018-10-02 03:18:08 +02:00
Georges Dupéron
ba6f2b39a9
Fixed typo (+ instead of *) 2018-10-02 03:18:06 +02:00
Georges Dupéron
6b0082dd68
Display warning: reproducible builds do not work on Windows or macos 2018-10-01 21:08:38 +02:00
Georges Dupéron
1b36f8e98b
Merge branch 'reproducible-build-windows' (unfinished) 2018-10-01 21:08:38 +02:00
Georges Dupéron
b8941befe8
Merge branch 'reproducible-build-macos' (does not work) 2018-10-01 21:08:37 +02:00
Georges Dupéron
76ef64b5be
Use faketime or datefudge if the former is not installed 2018-10-01 21:08:37 +02:00
Georges Dupéron
24fe32c3b5
Circumvent the fact that faketime does not work on system binaries in macos 2018-10-01 21:08:37 +02:00
Georges Dupéron
6f5522bdcb
Installed datefudge on Appveyor via Cygwin 2018-10-01 21:08:37 +02:00
Georges Dupéron
60a7f054b3
appveyor: clone_depth is unnecessary if shallow_clone is used 2018-10-01 21:08:37 +02:00
Georges Dupéron
fce3b9b6b3
Merge branch 'reproducible-build-linux' 2018-10-01 21:08:36 +02:00
Georges Dupéron
07e12d3062
cp -T option does not exist on on macos 2018-10-01 21:08:36 +02:00
Georges Dupéron
08901554b0
Screenshots directory is now an option, to avoid conflicts with the reproducible build's make clean 2018-10-01 21:08:36 +02:00
Georges Dupéron
2e9ce6fc26
macos: date conversion needs different flags than in Linux 2018-10-01 21:08:36 +02:00
Georges Dupéron
00bfba9251
macos: mkisofs includes the path to its input as extra unwanted directories in the iso 2018-10-01 21:08:36 +02:00
Georges Dupéron
634a4a0ff3
Omit the build directory path from os.offsets.hex
The build directory is not the same for the normal build and the build made to check for reproducibility.
2018-10-01 21:08:35 +02:00
Georges Dupéron
94deee14b0
Avoid copying iso_files as a subfolder of an existing iso_files.tmp if the latter was not cleaned correctly 2018-10-01 21:08:35 +02:00
Georges Dupéron
01ea5827b2
Added COMMIT_TIMESTAMP_ISO_8601 as a Makefile parameter 2018-10-01 21:08:35 +02:00
Georges Dupéron
7dcdca9649
Check that the build is reproducible 2018-10-01 21:08:35 +02:00
Georges Dupéron
01d3ae7544
Made the build directory configurable 2018-10-01 21:08:35 +02:00
Georges Dupéron
cd866fb6e7
Closes #42: Reproducible builds
232 initial number of different lines in the hexadecimal dump.
160 (recursive touch in iso_files)
108 (touch os.32k)
46  (faketime on mkisofs)
38  (faketime on mformat)
32  (faketime on mcopy)
0   (explicit partition and disk GUIDs)
2018-10-01 21:08:34 +02:00
Georges Dupéron
282f241459
Missing dependency in offsets calculation (could cause race condition when building with -j) 2018-10-01 21:08:30 +02:00
15 changed files with 644 additions and 418 deletions

4
.gitignore vendored
View File

@ -96,6 +96,7 @@
/build/test_pass/emu_virtualbox
/build/test_pass/noemu_fat12_contents
/build/test_pass/noemu_os.reasm
/build/test_pass/noemu_reproducible_build
/build/test_pass/noemu_sizes
/build/test_pass/noemu_zip
/build/test_pass/sudo_fat12_mount
@ -117,3 +118,6 @@
/deploy-screenshots/virtualbox-anim.gif
/deploy-screenshots/virtualbox.png
/os.bat
/utils/mcopy
/utils/mformat
/utils/mkisofs

View File

@ -22,6 +22,7 @@ addons:
- nasm
- mtools
- genisoimage
- faketime
matrix:
include:
@ -60,6 +61,7 @@ matrix:
- brew install mtools
- brew cask install gdisk
- brew install imagemagick # --with-x11
- brew install libfaketime
script: make all test/noemu test/macos
- os: linux
env: MODE=qemu-system-i386-floppy
@ -119,6 +121,7 @@ install:
- import --version || test "$MODE" = osx-test # import --version has a non-zero exit code on the Travis installation
- Xvfb -help
- ssh -V
- faketime --version
- ./utils/install-sikulix.sh
- export PATH="$HOME/sikulix/:$PATH"
- if test "$MODE" = virtualbox -o "$MODE" = self-test; then ./utils/install-virtualbox.sh; fi

549
Makefile
View File

@ -1,462 +1,185 @@
####################################
# Makefile configuration variables #
OS_FILENAME = os.bat
BUILD_DIR = build
SCREENSHOTS_DIR = deploy-screenshots
COMMIT_TIMESTAMP_ISO_8601 = $$(git log -1 --pretty=format:%ad --date=iso8601-strict)
####################################
MAKEFLAGS = --warn-undefined-variables
SHELL = bash -euET -o pipefail -c
.SECONDEXPANSION:
os_filename = os.bat
Makefiles = Makefile Makefile.example-os Makefile.test-example-os
os_filename = ${OS_FILENAME}
bld = ${BUILD_DIR}
screenshots = ${SCREENSHOTS_DIR}
tests_emu = test/qemu-system-i386-floppy test/qemu-system-i386-cdrom test/qemu-system-arm test/virtualbox test/bochs test/gui-sh test/dosbox
tests_requiring_sudo = test/fat12_mount test/iso_mount
tests_noemu = test/zip test/os.reasm test/sizes test/fat12_contents
tests_noemu = test/zip test/os.reasm test/sizes test/fat12_contents test/reproducible_build
offset_names = bytes_os_size \
bytes_mbr_start \
bytes_mbr_end \
bytes_header_32k_start \
bytes_header_32k_end \
bytes_iso_start \
bytes_iso_end \
bytes_fat12_start \
bytes_fat12_end \
bytes_gpt_mirror_start \
bytes_gpt_mirror_end \
bytes_zip_start \
bytes_zip_end
# We truncate the timezone, because the Darwin version of date seems to lack
# the %:z format (for ±HH:MM timezone).
define date_command
if test "$$(uname -s)" = Darwin; then \
date -j -f %Y-%m-%dT%H:%M:%S $$(echo ${1} | cut -c 1-19) ${2}; \
else \
date -d ${1} ${2}; \
fi
endef
commit_timestamp = "$$(${call date_command,"${COMMIT_TIMESTAMP_ISO_8601}",'+%Y%m%d%H%m.%S'})"
commit_timestamp_iso_8601 = ${COMMIT_TIMESTAMP_ISO_8601}
more_offset_names = ${offset_names} \
bytes_fat12_size \
bytes_gpt_mirror_size \
bytes_header_32k_size \
bytes_iso_size \
bytes_zip_size \
sectors_fat12_size \
sectors_fat12_start \
sectors_gpt_mirror_size \
sectors_iso_size \
sectors_os_size \
sectors_zip_size \
tracks_fat12_size \
tracks_gpt_mirror_size \
tracks_iso_size \
tracks_os_size \
tracks_zip_size
reproducible_os_filename="${bld}/reproduced_$$(basename "${os_filename}")"
more_offset_dec = ${more_offset_names:%=build/offsets/%.dec}
more_offset_hex = ${more_offset_names:%=build/offsets/%.hex}
built_files += ${bld}/check_makefile \
${bld}/check_makefile_targets \
${bld}/check_makefile_w_arnings \
${bld}/makefile_built_directories \
${bld}/makefile_built_files \
${bld}/makefile_database \
${bld}/makefile_database_files \
${bld}/makefile_file_targets \
${bld}/makefile_non_file_targets \
${bld}/makefile_phony \
${bld}/makefile_targets \
${bld}/makefile_w_arnings
# + os.arm.disasm
# + os.reasm.disasm
built_files = ${os_filename} \
build/check_makefile \
build/check_makefile_targets \
build/check_makefile_w_arnings \
build/checkerboard_800x600.xbm \
build/checkerboard_1024x768.png \
build/makefile_built_directories \
build/makefile_built_files \
build/makefile_database \
build/makefile_database_files \
build/makefile_file_targets \
build/makefile_non_file_targets \
build/makefile_phony \
build/makefile_targets \
build/makefile_w_arnings \
build/os.ndisasm.disasm \
build/os.reasm.asm \
build/os.reasm \
build/os.file \
build/os.gdisk \
build/os.zip \
build/os.zip.adjusted \
build/os.iso \
build/os.32k \
build/os.fat12 \
build/os.offsets.hex \
build/os.offsets.dec \
build/os.hex_with_offsets \
build/iso_files/os.zip \
build/iso_files/boot/iso_boot.sys \
build/bochsrc \
build/bochscontinue \
build/twm_cfg \
build/virtualbox.img \
${more_offset_dec} \
${more_offset_hex} \
${tests_emu:test/%=build/test_pass/emu_%} \
${tests_noemu:test/%=build/test_pass/noemu_%} \
${tests_requiring_sudo:test/%=build/test_pass/sudo_%} \
${tests_emu:test/%=deploy-screenshots/%.png} \
${tests_emu:test/%=deploy-screenshots/%-anim.gif}
include Makefile.example-os
include Makefile.test-example-os
built_directories = build/iso_files/boot build/iso_files build/offsets build/mnt_fat12 build/mnt_iso build/test_pass deploy-screenshots
more_built_directories = ${built_directories} build
os_image_size_kb = 1440
os_partition_start_sectors = 3
os_partition_size_sectors = 717 # 720 - start
# CHS parameters for 1.44 MB floppy disk
os_floppy_chs_h = 2
os_floppy_chs_s = 9
more_built_directories = ${built_directories} ${bld}
.PHONY: all
# all: os.arm.disasm
all: ${os_filename} build/os.ndisasm.disasm build/os.reasm.asm build/os.file build/os.gdisk build/os.offsets.hex build/os.offsets.dec build/os.hex_with_offsets .gitignore build/check_makefile ${more_offset_dec} ${more_offset_hex}
all: .gitignore \
${bld}/check_makefile
build/makefile_w_arnings: | $${@D}
${bld}/makefile_w_arnings: | $${@D}
${built_files}: | $${@D}
build/makefile_w_arnings: Makefile
@unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; make -n --warn-undefined-variables test 2>$@ 1>/dev/null || make -n --warn-undefined-variables test
${bld}/makefile_w_arnings: ${Makefiles}
@unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \
make -n --warn-undefined-variables \
OS_FILENAME=${OS_FILENAME} \
BUILD_DIR=${BUILD_DIR} \
SCREENSHOTS_DIR=${SCREENSHOTS_DIR} \
COMMIT_TIMESTAMP_ISO_8601=${COMMIT_TIMESTAMP_ISO_8601} \
test 2>$@ 1>/dev/null \
|| cat $@
# Check that the file build/makefile_w_arnings is present, and that it does not contain the string "warn".
build/check_makefile_w_arnings: build/makefile_w_arnings
@cat build/makefile_w_arnings > /dev/null && (! grep -i warn $<) && touch $@
# Check that the file ${bld}/makefile_w_arnings is present, and that it does not contain the string "warn".
${bld}/check_makefile_w_arnings: ${bld}/makefile_w_arnings
@cat ${bld}/makefile_w_arnings > /dev/null && (! grep -i warn $<) && touch $@
# Check that the declared list of built files matches the list of targets extracted from the Makefile.
build/check_makefile_targets: build/makefile_built_files build/makefile_file_targets build/check_makefile_w_arnings
@diff build/makefile_built_files build/makefile_file_targets && touch $@
${bld}/check_makefile_targets: ${bld}/makefile_built_files ${bld}/makefile_file_targets ${bld}/check_makefile_w_arnings
@diff ${bld}/makefile_built_files ${bld}/makefile_file_targets && touch $@
build/check_makefile: build/check_makefile_w_arnings build/check_makefile_targets
${bld}/check_makefile: ${bld}/check_makefile_w_arnings ${bld}/check_makefile_targets
@touch $@
build/makefile_database: Makefile build/check_makefile_w_arnings
@unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; make -rpn | sed -n -e '/^# Make data base,/,$$p' > $@
${bld}/makefile_database: ${Makefiles} ${bld}/check_makefile_w_arnings
@unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \
make -rpn \
OS_FILENAME=${OS_FILENAME} \
BUILD_DIR=${BUILD_DIR} \
SCREENSHOTS_DIR=${SCREENSHOTS_DIR} \
COMMIT_TIMESTAMP_ISO_8601=${COMMIT_TIMESTAMP_ISO_8601} \
| sed -n -e '/^# Make data base,/,$$p' > $@
build/makefile_database_files: build/makefile_database build/check_makefile_w_arnings
${bld}/makefile_database_files: ${bld}/makefile_database ${bld}/check_makefile_w_arnings
@sed -n -e '/^# Files$$/,/^# files hash-table stats:$$/p' $< > $@
build/makefile_built_directories: build/check_makefile_w_arnings
${bld}/makefile_built_directories: ${bld}/check_makefile_w_arnings
@echo ${more_built_directories} | tr ' ' '\n' | grep -v '^[[:space:]]*$$' | sort > $@
build/makefile_built_files: build/check_makefile_w_arnings
${bld}/makefile_built_files: ${bld}/check_makefile_w_arnings
@echo ${built_files} | tr ' ' '\n' | grep -v '^[[:space:]]*$$' | sort > $@
build/makefile_phony: build/makefile_database_files build/check_makefile_w_arnings
${bld}/makefile_phony: ${bld}/makefile_database_files ${bld}/check_makefile_w_arnings
@sed -n -e 's/^\.PHONY: \(.*\)$$/\1/p' $< | tr ' ' '\n' | grep -v '^[[:space:]]*$$' | sort > $@
build/makefile_targets: build/makefile_database_files build/check_makefile_w_arnings
${bld}/makefile_targets: ${bld}/makefile_database_files ${bld}/check_makefile_w_arnings
@grep -E -v '^([[:space:]]|#|\.|$$|^[^:]*:$$)' $< | grep '^[^ :]*:' | sed -e 's|^\([^:]*\):.*$$|\1|' | sort > $@
build/makefile_non_file_targets: build/makefile_phony build/makefile_built_directories build/check_makefile_w_arnings
@cat build/makefile_phony build/makefile_built_directories | sort > $@
${bld}/makefile_non_file_targets: ${bld}/makefile_phony ${bld}/makefile_built_directories ${bld}/check_makefile_w_arnings
@cat ${bld}/makefile_phony ${bld}/makefile_built_directories | sort > $@
build/makefile_file_targets: build/makefile_non_file_targets build/makefile_targets build/check_makefile_w_arnings
@comm -23 build/makefile_targets build/makefile_non_file_targets > $@
${bld}/makefile_file_targets: ${bld}/makefile_non_file_targets ${bld}/makefile_targets ${bld}/check_makefile_w_arnings
@comm -23 ${bld}/makefile_targets ${bld}/makefile_non_file_targets > $@
${built_directories}: build/check_makefile
${more_built_directories}: Makefile
${built_directories}: ${bld}/check_makefile
${more_built_directories}: ${Makefiles}
mkdir -p $@ && touch $@
# 32k header of the ISO9660 image
build/os.32k: example-os/os.asm build/check_makefile
nasm -w+macro-params -w+macro-selfref -w+orphan-labels -w+gnu-elf-extensions -o $@ $<
build/os.iso: build/iso_files/os.zip build/iso_files/boot/iso_boot.sys build/check_makefile
mkisofs \
--input-charset utf-8 \
-rock \
-joliet \
-eltorito-catalog boot/boot.cat \
-eltorito-boot boot/iso_boot.sys \
-no-emul-boot \
-boot-load-size 4 \
-pad \
-output $@ \
./build/iso_files/
# Layout:
# MBR; GPT; UNIX sh & MS-DOS batch scripts; ISO9660; FAT12; GPT mirror; ZIP
define offset
tmp_${1} = ${3}
build/offsets/${1}.dec: $${tmp_${1}:%=build/offsets/%.dec} ${4} build/check_makefile
echo $$$$(( ${2} )) | tee $$@
${1} = $$$$(cat build/offsets/${1}.dec)
dep_${1} = build/offsets/${1}.dec
endef
define div_round_up
( ( ( ${1} ) + ( ${2} ) - 1 ) / ( ${2} ) )
endef
sector_size = 512
# should be exact (TODO: make a check)
${eval ${call offset,bytes_os_size, $${os_image_size_kb} * 1024,, }}
${eval ${call offset,sectors_os_size, $${bytes_os_size} / $${sector_size}, bytes_os_size,}}
${eval ${call offset,tracks_os_size, $${sectors_os_size} / $${os_floppy_chs_s}, sectors_os_size,}}
# round up
${eval ${call offset,bytes_iso_size, $$$$(utils/file-length.sh -c build/os.iso), ,build/os.iso}}
${eval ${call offset,sectors_iso_size, ${call div_round_up,$${bytes_iso_size},$${sector_size}}, bytes_iso_size,}}
${eval ${call offset,tracks_iso_size, ${call div_round_up,$${sectors_iso_size},$${os_floppy_chs_s}}, sectors_iso_size,}}
# round up
${eval ${call offset,bytes_zip_size, $$$$(utils/file-length.sh -c build/os.zip), ,build/os.zip}}
${eval ${call offset,sectors_zip_size, ${call div_round_up,$${bytes_zip_size},$${sector_size}}, bytes_zip_size,}}
${eval ${call offset,tracks_zip_size, ${call div_round_up,$${sectors_zip_size},$${os_floppy_chs_s}}, sectors_zip_size,}}
# round up
${eval ${call offset,sectors_gpt_mirror_size, 33,, }}
${eval ${call offset,tracks_gpt_mirror_size, ${call div_round_up,$${sectors_gpt_mirror_size},$${os_floppy_chs_s}}, sectors_gpt_mirror_size,}}
# allocate the remaining sectors to the FAT, aligned on tracks
${eval ${call offset,tracks_fat12_size, $${tracks_os_size} - $${tracks_iso_size} - $${tracks_gpt_mirror_size} - $${tracks_zip_size}, tracks_os_size tracks_iso_size tracks_gpt_mirror_size tracks_zip_size,}}
${eval ${call offset,sectors_fat12_size,$${tracks_fat12_size} * $${os_floppy_chs_s}, tracks_fat12_size,}}
# zip should probably have its end aligned, not its start
${eval ${call offset,bytes_zip_start, $${bytes_os_size} - $${bytes_zip_size}, bytes_os_size bytes_zip_size,}}
${eval ${call offset,bytes_mbr_start, 0,,}}
${eval ${call offset,bytes_mbr_end, 512,,}}
${eval ${call offset,bytes_header_32k_start, 0,,}}
${eval ${call offset,bytes_header_32k_end, 32 * 1024,,}}
${eval ${call offset,bytes_header_32k_size, $${bytes_header_32k_end} - $${bytes_header_32k_start}, bytes_header_32k_end bytes_header_32k_start,}}
${eval ${call offset,bytes_iso_start, 32 * 1024,,}}
${eval ${call offset,bytes_iso_end, $${sectors_iso_size} * $${sector_size}, sectors_iso_size,}}
${eval ${call offset,bytes_fat12_start, $${tracks_iso_size} * $${os_floppy_chs_s} * $${sector_size}, tracks_iso_size,}}
${eval ${call offset,sectors_fat12_start, $${bytes_fat12_start} / $${sector_size}, bytes_fat12_start,}}
${eval ${call offset,bytes_fat12_size, $${sectors_fat12_size} * $${sector_size}, sectors_fat12_size,}}
${eval ${call offset,bytes_fat12_end, $${bytes_fat12_start} + $${bytes_fat12_size}, bytes_fat12_start bytes_fat12_size,}}
# It is probably not necessary to align the GPT mirror end on a track boundary.
${eval ${call offset,bytes_gpt_mirror_size, $${sectors_gpt_mirror_size} + $${sector_size}, sectors_gpt_mirror_size,}}
${eval ${call offset,bytes_gpt_mirror_end, $${bytes_fat12_end} + $${bytes_gpt_mirror_size}, bytes_fat12_end bytes_gpt_mirror_size,}}
${eval ${call offset,bytes_gpt_mirror_start, $${bytes_gpt_mirror_end} - $${bytes_gpt_mirror_size}, bytes_gpt_mirror_end bytes_gpt_mirror_size,}}
${eval ${call offset,bytes_zip_end, $${bytes_os_size},, }}
os_fat12_partition = "$@@@${bytes_fat12_start}"
build/os.fat12: build/os.zip ${dep_bytes_fat12_size} ${dep_bytes_fat12_start} ${dep_sectors_os_size} build/check_makefile
set -x; dd if=/dev/zero bs=${sector_size} count=${sectors_os_size} of=$@
set -x; mformat -v "Example OS" \
-T ${sectors_fat12_size} \
-h ${os_floppy_chs_h} \
-s ${os_floppy_chs_s} \
-i ${os_fat12_partition}
set -x; mcopy -i ${os_fat12_partition} build/os.zip "::os.zip"
build/iso_files/os.zip: build/os.zip build/check_makefile
# TODO: make it so that the various file formats are mutual quines:
# * the ISO should contain the original file
# * the ZIP should contain the original file
# * the FAT12 should contain the original file
cp $< $@
# 4 sectors loaded when booting from optical media (CD-ROM, …):
build/iso_files/boot/iso_boot.sys: build/os.32k build/check_makefile
# TODO: this copy of the (or alternate) bootsector should contain a Boot Information Table,
# see https://wiki.osdev.org/El-Torito#A_BareBones_Boot_Image_with_Boot_Information_Table
dd if=$< bs=512 count=4 of=$@
build/os.zip: build/os.32k build/check_makefile
zip $@ $<
build/os.zip.adjusted: build/os.zip ${dep_bytes_zip_start} build/check_makefile
# TODO: the ZIP file can end with a variable-length comment, this would allow us to hide the GPT mirrors.
set -x; dd if=/dev/zero bs=1 count=${bytes_zip_start} of=$@
cat $< >> $@
zip --adjust-sfx $@
${os_filename}: build/os.32k build/os.iso build/os.fat12 build/os.zip.adjusted \
${dep_bytes_header_32k_start} \
${dep_bytes_header_32k_size} \
${dep_bytes_fat12_start} \
${dep_bytes_fat12_size} \
${dep_bytes_gpt_mirror_start} \
${dep_bytes_gpt_mirror_end} \
${dep_sectors_fat12_start} \
${dep_sectors_fat12_size} \
${dep_bytes_zip_start} \
build/check_makefile
rm -f $@
# start with the .iso
cp build/os.iso $@
# splice in the first 32k (bootsector and partition table)
set -x; dd skip=${bytes_header_32k_start} seek=${bytes_header_32k_start} bs=1 count=${bytes_header_32k_size} conv=notrunc if=build/os.32k of=$@
# splice in fat12
set -x; dd skip=${bytes_fat12_start} seek=${bytes_fat12_start} bs=1 count=${bytes_fat12_size} conv=notrunc if=build/os.fat12 of=$@
# pad with zeroes to prepare for GPT table
set -x; dd if=/dev/zero seek=$$((${bytes_gpt_mirror_end} - 1 )) bs=1 count=1 conv=notrunc of=$@
# patch the partition table
# Thanks to https://wiki.gentoo.org/wiki/Hybrid_partition_table for showing that gdisk can be used to make a hybrid MBR / GPT.
# gdisk commands:
# * Delete (the only) partition, eXpert mode, sector aLignment = 1, back to Main menu,
# * New partition (number = 1, start sector, end sector, type 0700)
# * Recovery and transformation options, make Hybrid,
# * add GPT partition #1 to the hybrid MBR, do Not put the EFI partition first,
# * MBR partition type=0x01, bootable=Yes, do Not add extra partitions,
# * Print GPT, print MBR, Write, Proceed.
(if test "$$(uname -o)" = "Cygwin"; then printf "Y\n"; fi; \
printf "d\nx\nl\n1\nm\nn\n1\n${sectors_fat12_start}\n${sectors_fat12_size}\n0700\n"; \
printf "r\nh\n"; \
printf "1\nN\n"; \
printf "01\nY\nN\n"; \
printf "p\no\nw\nY\n") | while read str; do echo "$$str"; printf "\033[1;33m%s\033[m\n" "$$str" >&2; sleep 0.01; done | gdisk $@
# Inject MS-DOS newlines (CR+LF) and comments (":: ") in the GUID field of unused partition table entries,
# so that the part that is to be skipped by MS-DOS does not form a line longer than the MS-DOS maximum
# line length (8192 excluding CR+LF). $i below is the partition entry number, starting from 1
# The numbers 55 and 118 are arbitrarily chosen so that the space between two CR+LF is less than 8192.
for i in 55 118; do \
printf "\r\n:: %02x" $$i | dd bs=1 seek=$$(( 1024 + ( ($$i) - 1) * 128 + 16)) count=7 conv=notrunc of=$@; \
printf "\r\n:: %02x" $$i | dd bs=1 seek=$$(( ${bytes_gpt_mirror_start} + ( ($$i) - 1) * 128 + 16)) count=7 conv=notrunc of=$@; \
done
# splice in zip at the end
set -x; dd skip=${bytes_zip_start} seek=${bytes_zip_start} bs=1 conv=notrunc if=build/os.zip.adjusted of=$@
chmod a+x-w $@
build/os.file: ${os_filename} build/check_makefile
file -kr $< > $@
build/os.gdisk: ${os_filename} build/check_makefile
# gdisk commands:
# * Print partition table
# * Recovery and transformation options
# * print prOtective MBR table
# * Quit
printf 'p\nr\no\nq\n' | gdisk $< | tee $@
build/os.offsets.hex: ${offset_names:%=build/offsets/%.hex} build/check_makefile
grep '^' ${offset_names:%=build/offsets/%.hex} | sed -e 's/:/: 0x/' | column -t > $@
build/os.offsets.dec: ${offset_names:%=build/offsets/%.dec} build/check_makefile
grep '^' ${offset_names:%=build/offsets/%.dec} | sed -e 's/:/: /' | column -t > $@
build/offsets/%.hex: build/offsets/%.dec
printf '%x\n' $$(cat $<) > $@
build/os.hex_with_offsets: ${os_filename} build/os.offsets.hex
hexdump -C $< \
| grep -E -e "($$(cat build/os.offsets.hex | cut -d '=' -f 2 | sed -e 's/^[[:space:]]*0x\(.*\).$$/^\10/' | tr '\n' '|')^)" --color=yes > $@
build/os.ndisasm.disasm: ${os_filename} utils/compact-ndisasm.sh build/check_makefile
./utils/compact-ndisasm.sh $< $@
build/os.reasm.asm: build/os.ndisasm.disasm build/check_makefile
sed -e 's/^[^ ]\+ \+[^ ]\+ \+//' $< > $@
build/test_pass/noemu_%.reasm build/%.reasm: build/%.reasm.asm ${os_filename} utils/compact-ndisasm.sh build/check_makefile
# For now ignore this test, since we cannot have a reliable re-assembly of arbitrary data.
touch build/test_pass/noemu_$*.reasm build/$*.reasm
# nasm $< -o $@
# @echo "diff $@ ${os_filename}"
# @diff $@ ${os_filename} \
# && echo "Re-assembled file is identical to ${os_filename}" \
# || (./utils/compact-ndisasm.sh $@ build/os.reasm.disasm; \
# echo "Re-assembled file is different from ${os_filename}. Use meld build/os.ndisasm.disasm build/os.reasm.disasm to see differences."; \
# exit 0)
#os.arm.disasm: ${os_filename} build/check_makefile
# arm-none-eabi-objdump --endian=little -marm -b binary -D --adjust-vma=0x8000 $< > $@
.PHONY: clean
clean: build/check_makefile
rm -f ${built_files}
for d in $$(echo ${more_built_directories} | tr ' ' '\n' | sort --reverse); do \
clean: clean_reproducible ${bld}/check_makefile
rm -f ${built_files} ${temp_files}
for d in $$(echo ${more_built_directories} ${temp_directories} | tr ' ' '\n' | sort --reverse); do \
if test -e "$$d"; then \
rmdir "$$d"; \
fi; \
done
.gitignore: build/check_makefile
.PHONY: clean_reproducible
clean_reproducible: ${bld}/check_makefile
# Calling make unconditionally would cause infinite recursion, so we
# first check whether there is anything to remove.
if test -d ${bld}/reproducible -o -f ${bld}/${reproducible_os_filename}; then \
unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \
make OS_FILENAME=${reproducible_os_filename} \
BUILD_DIR=${bld}/reproducible \
SCREENSHOTS_DIR=${bld}/reproducible/screenshots \
COMMIT_TIMESTAMP_ISO_8601=${COMMIT_TIMESTAMP_ISO_8601} \
clean; \
fi
.gitignore: ${bld}/check_makefile
for f in ${built_files}; do echo "/$$f"; done | sort > $@
.PHONY: test
test: ${tests_emu:test/%=build/test_pass/emu_%} \
${tests_noemu:test/%=build/test_pass/noemu_%} \
${tests_requiring_sudo:test/%=build/test_pass/sudo_%} \
all \
build/check_makefile
test: all \
${bld}/check_makefile
.PHONY: ${tests_emu}
${tests_emu}: build/test_pass/emu_$$(@F)
build/test_pass/emu_% deploy-screenshots/%.png deploy-screenshots/%-anim.gif: \
${os_filename} \
build/checkerboard_800x600.xbm \
utils/gui-wrapper.sh utils/ansi-screenshots/ansi_screenshot.sh utils/ansi-screenshots/to_ansi.sh \
test/%.sh \
build/check_makefile \
| build/test_pass deploy-screenshots
./utils/gui-wrapper.sh 800x600x24 ./test/$*.sh $<
touch build/test_pass/emu_$*
.PHONY: test/noemu
test/noemu: ${tests_noemu:test/%=build/test_pass/noemu_%} build/check_makefile
build/test_pass/noemu_zip: ${os_filename} build/check_makefile
unzip -t ${os_filename}
touch $@
build/test_pass/noemu_sizes: build/os.32k ${os_filename} build/check_makefile
test "$$(utils/file-length.sh -c build/os.32k)" = "$$((32*1024))"
test "$$(utils/file-length.sh -c ${os_filename})" = "$$((1440*1024))"
touch $@
# check that the fat filesystem has the correct contents
build/test_pass/noemu_fat12_contents: ${os_filename} ${dep_bytes_fat12_start} build/check_makefile
mdir -i "$<@@${bytes_fat12_start}" :: | grep -E "^os[[:space:]]+zip[[:space:]]+"
touch $@
.PHONY: test/requiring_sudo
test/requiring_sudo: ${tests_requiring_sudo:test/%=build/test_pass/sudo_%} build/check_makefile
# check that the fat filesystem can be mounted and has the correct contents
build/test_pass/sudo_fat12_mount: ${os_filename} ${dep_bytes_fat12_start} build/check_makefile | build/mnt_fat12
sudo umount build/mnt_fat12 || true
sudo mount -o loop,ro,offset=${bytes_fat12_start} $< build/mnt_fat12
ls -l build/mnt_fat12 | grep os.zip
sudo umount build/mnt_fat12
touch $@
build/test_pass/sudo_iso_mount: ${os_filename} build/check_makefile | build/mnt_iso
sudo umount build/mnt_iso || true
grep '^' build/offsets/* # debug failure to mount the ISO9660 filesystem
(sudo mount -o loop,ro $< build/mnt_iso) || true
dmesg | tail # debug failure to mount the ISO9660 filesystem
hexdump -C os.bat
ls -l build/mnt_iso | grep os.zip
sudo umount build/mnt_iso
sudo mount -o loop,ro $< build/mnt_iso
sudo umount build/mnt_iso
touch $@
.PHONY: test/macos
test/macos: all test/noemu test/macos-sh test/macos-sh-x11
.PHONY: test/macos-sh-x11
test/macos-sh-x11:
sudo mkdir -p /tmp/.X11-unix
sudo chmod a+rwxt /tmp/.X11-unix
xvfb :42 & \
sleep 5; \
DISPLAY=:42 xterm -e ./os.bat & \
sleep 5; \
# DISPLAY=:42 import -window root deploy-screenshots/macos-sh-x11.png
screencapture deploy-screenshots/macos-sh-x11-screencapture.png
.PHONY: test/macos-sh
test/macos-sh: build/check_makefile \
build/checkerboard_1024x768.png \
| deploy-screenshots
osascript -e 'tell app "Terminal" to do script "'"$$PWD"'/os.bat"'
sleep 2
osascript -e 'tell app "Terminal" to activate'
# DEBUG: ${bld}/os.hex_with_offsets ${bld}/os.offsets.hex
${bld}/test_pass/noemu_reproducible_build: ${os_filename} ${bld}/os.hex_with_offsets ${bld}/check_makefile
# Let some time pass so that any timestamp that may affect the result changes.
sleep 5
(date +%n && sleep 0.2 && date +%n) || true
screencapture deploy-screenshots/screencapture-os-bat.png
./utils/gui-wrapper-mac.sh 1024x768x24 ./test/gui-sh-mac.sh ${os_filename}
# See https://wiki.osdev.org/EFI#Emulation to emulate an UEFI system with qemu, to test the EFI boot from hdd / cd / fd (?).
# Create checkerboard background
build/checkerboard_%.png: build/check_makefile
convert -size "$*" \
tile:pattern:checkerboard \
-auto-level +level-colors 'gray(192),gray(128)' \
$@
build/checkerboard_%.xbm: build/check_makefile
convert -size "$*" \
tile:pattern:checkerboard \
-auto-level \
$@
# Temporary files
build/bochsrc build/bochscontinue build/twm_cfg build/virtualbox.img:
# TODO: try to see if we can re-enable some of these variables without
# causing problems on macos.
unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \
make OS_FILENAME=${reproducible_os_filename} \
BUILD_DIR=${bld}/reproducible \
SCREENSHOTS_DIR=${bld}/reproducible/screenshots \
COMMIT_TIMESTAMP_ISO_8601=${COMMIT_TIMESTAMP_ISO_8601} \
clean
unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \
make OS_FILENAME=${reproducible_os_filename} \
BUILD_DIR=${bld}/reproducible \
SCREENSHOTS_DIR=${bld}/reproducible/screenshots \
COMMIT_TIMESTAMP_ISO_8601=${COMMIT_TIMESTAMP_ISO_8601} \
${reproducible_os_filename} \
${bld}/reproducible/os.hex_with_offsets
# Check that the second build produced the same file.
if ! diff ${os_filename} ${reproducible_os_filename}; then \
diff ${bld}/os.hex_with_offsets ${bld}/reproducible/os.hex_with_offsets || true; \
if test "$$(uname -s)" = Darwin -o "$$(uname -o)" = "Cygwin"; then \
for i in `seq 5`; do \
printf '\033[1;31m########################################################\033[m'; \
done; \
echo "REPRODUCIBLE BUILDS ARE UNSUPPORTED ON MACOS AND WINDOWS"; \
for i in `seq 5`; do \
printf '\033[1;31m########################################################\033[m'; \
done; \
else \
exit 1; \
fi; \
fi
unset MAKEFLAGS MAKELEVEL MAKE_TERMERR MFLAGS; \
make OS_FILENAME=${reproducible_os_filename} \
BUILD_DIR=${bld}/reproducible \
SCREENSHOTS_DIR=${bld}/reproducible/screenshots \
COMMIT_TIMESTAMP_ISO_8601=${COMMIT_TIMESTAMP_ISO_8601} \
clean
touch $@

308
Makefile.example-os Normal file
View File

@ -0,0 +1,308 @@
offset_names = bytes_os_size \
bytes_mbr_start \
bytes_mbr_end \
bytes_header_32k_start \
bytes_header_32k_end \
bytes_iso_start \
bytes_iso_end \
bytes_fat12_start \
bytes_fat12_end \
bytes_gpt_mirror_start \
bytes_gpt_mirror_end \
bytes_zip_start \
bytes_zip_end
more_offset_names = ${offset_names} \
bytes_fat12_size \
bytes_gpt_mirror_size \
bytes_header_32k_size \
bytes_iso_size \
bytes_zip_size \
sectors_fat12_size \
sectors_fat12_start \
sectors_gpt_mirror_size \
sectors_iso_size \
sectors_os_size \
sectors_zip_size \
tracks_fat12_size \
tracks_gpt_mirror_size \
tracks_iso_size \
tracks_os_size \
tracks_zip_size
more_offset_dec = ${more_offset_names:%=${bld}/offsets/%.dec}
more_offset_hex = ${more_offset_names:%=${bld}/offsets/%.hex}
# + os.arm.disasm
# + os.reasm.disasm
built_files += ${os_filename} \
${bld}/os.ndisasm.disasm \
${bld}/os.reasm.asm \
${bld}/os.reasm \
${bld}/os.file \
${bld}/os.gdisk \
${bld}/os.zip \
${bld}/os.zip.adjusted \
${bld}/os.iso \
${bld}/os.32k \
${bld}/os.fat12 \
${bld}/os.offsets.hex \
${bld}/os.offsets.dec \
${bld}/os.hex_with_offsets \
${bld}/iso_files/os.zip \
${bld}/iso_files/boot/iso_boot.sys \
${more_offset_dec} \
${more_offset_hex} \
# Temporary copies used to adjust timestamps for reproducible builds.
# These are normally created and deleted within a single target, but
# could remain if make is interrupted during the build.
temp_files = ${bld}/iso_files.tmp/os.zip \
${bld}/iso_files.tmp/boot/iso_boot.sys
temp_directories = ${bld}/iso_files.tmp \
${bld}/iso_files.tmp/boot/
built_directories += ${bld}/iso_files/boot \
${bld}/iso_files \
${bld}/offsets
os_image_size_kb = 1440
os_partition_start_sectors = 3
os_partition_size_sectors = 717 # 720 - start
# CHS parameters for 1.44 MB floppy disk
os_floppy_chs_h = 2
os_floppy_chs_s = 9
all: ${os_filename} \
${bld}/os.ndisasm.disasm \
${bld}/os.reasm.asm \
${bld}/os.file \
${bld}/os.gdisk \
${bld}/os.offsets.hex \
${bld}/os.offsets.dec \
${bld}/os.hex_with_offsets \
${more_offset_dec} \
${more_offset_hex} \
${bld}/check_makefile
# 32k header of the ISO9660 image
${bld}/os.32k: example-os/os.asm ${bld}/check_makefile
nasm -w+macro-params -w+macro-selfref -w+orphan-labels -w+gnu-elf-extensions -o $@ $<
# Circumvent the fact that faketime does not work on system binaries in macos
./utils/mkisofs ./utils/mformat ./utils/mcopy: ${bld}/check_makefile # TODO: depend on the mkisofs binary
cp $$(which $$(basename $@)) $@
chmod u+x $@
cp_T_option = $$(if test "$$(uname -s)" = Darwin; then echo ''; else echo '-T'; fi)
${bld}/os.iso: ${bld}/iso_files/os.zip ${bld}/iso_files/boot/iso_boot.sys ./utils/mkisofs ${bld}/check_makefile
! test -d ${bld}/iso_files.tmp
cp -a ${cp_T_option} -- ${bld}/iso_files ${bld}/iso_files.tmp
find ${bld}/iso_files.tmp -depth -exec touch -t ${commit_timestamp} '{}' ';'
UTILS="$$PWD/utils" ./utils/faketime.sh ${commit_timestamp_iso_8601} sh -c '(cd ./${bld}/iso_files.tmp/ && "$$UTILS/mkisofs" \
--input-charset utf-8 \
-rock \
-joliet \
-eltorito-catalog boot/boot.cat \
-eltorito-boot boot/iso_boot.sys \
-no-emul-boot \
-boot-load-size 4 \
-pad \
-output ../os.iso \
.)'
rm -- ${bld}/iso_files.tmp/os.zip \
${bld}/iso_files.tmp/boot/iso_boot.sys
rmdir ${bld}/iso_files.tmp/boot/
rmdir ${bld}/iso_files.tmp/
# Layout:
# MBR; GPT; UNIX sh & MS-DOS batch scripts; ISO9660; FAT12; GPT mirror; ZIP
define offset
tmp_${1} = ${3}
${bld}/offsets/${1}.dec: $${tmp_${1}:%=${bld}/offsets/%.dec} ${4} ${bld}/check_makefile
echo $$$$(( ${2} )) | tee $$@
${1} = $$$$(cat ${bld}/offsets/${1}.dec)
dep_${1} = ${bld}/offsets/${1}.dec
endef
define div_round_up
( ( ( ${1} ) + ( ${2} ) - 1 ) / ( ${2} ) )
endef
sector_size = 512
# should be exact (TODO: make a check)
${eval ${call offset,bytes_os_size, $${os_image_size_kb} * 1024,, }}
${eval ${call offset,sectors_os_size, $${bytes_os_size} / $${sector_size}, bytes_os_size,}}
${eval ${call offset,tracks_os_size, $${sectors_os_size} / $${os_floppy_chs_s}, sectors_os_size,}}
# round up
${eval ${call offset,bytes_iso_size, $$$$(utils/file-length.sh -c ${bld}/os.iso), ,${bld}/os.iso}}
${eval ${call offset,sectors_iso_size, ${call div_round_up,$${bytes_iso_size},$${sector_size}}, bytes_iso_size,}}
${eval ${call offset,tracks_iso_size, ${call div_round_up,$${sectors_iso_size},$${os_floppy_chs_s}}, sectors_iso_size,}}
# round up
${eval ${call offset,bytes_zip_size, $$$$(utils/file-length.sh -c ${bld}/os.zip), ,${bld}/os.zip}}
${eval ${call offset,sectors_zip_size, ${call div_round_up,$${bytes_zip_size},$${sector_size}}, bytes_zip_size,}}
${eval ${call offset,tracks_zip_size, ${call div_round_up,$${sectors_zip_size},$${os_floppy_chs_s}}, sectors_zip_size,}}
# round up
${eval ${call offset,sectors_gpt_mirror_size, 33,, }}
${eval ${call offset,tracks_gpt_mirror_size, ${call div_round_up,$${sectors_gpt_mirror_size},$${os_floppy_chs_s}}, sectors_gpt_mirror_size,}}
# allocate the remaining sectors to the FAT, aligned on tracks
${eval ${call offset,tracks_fat12_size, $${tracks_os_size} - $${tracks_iso_size} - $${tracks_gpt_mirror_size} - $${tracks_zip_size}, tracks_os_size tracks_iso_size tracks_gpt_mirror_size tracks_zip_size,}}
${eval ${call offset,sectors_fat12_size,$${tracks_fat12_size} * $${os_floppy_chs_s}, tracks_fat12_size,}}
# zip should probably have its end aligned, not its start
${eval ${call offset,bytes_zip_start, $${bytes_os_size} - $${bytes_zip_size}, bytes_os_size bytes_zip_size,}}
${eval ${call offset,bytes_mbr_start, 0,,}}
${eval ${call offset,bytes_mbr_end, 512,,}}
${eval ${call offset,bytes_header_32k_start, 0,,}}
${eval ${call offset,bytes_header_32k_end, 32 * 1024,,}}
${eval ${call offset,bytes_header_32k_size, $${bytes_header_32k_end} - $${bytes_header_32k_start}, bytes_header_32k_end bytes_header_32k_start,}}
${eval ${call offset,bytes_iso_start, 32 * 1024,,}}
${eval ${call offset,bytes_iso_end, $${sectors_iso_size} * $${sector_size}, sectors_iso_size,}}
${eval ${call offset,bytes_fat12_start, $${tracks_iso_size} * $${os_floppy_chs_s} * $${sector_size}, tracks_iso_size,}}
${eval ${call offset,sectors_fat12_start, $${bytes_fat12_start} / $${sector_size}, bytes_fat12_start,}}
${eval ${call offset,bytes_fat12_size, $${sectors_fat12_size} * $${sector_size}, sectors_fat12_size,}}
${eval ${call offset,bytes_fat12_end, $${bytes_fat12_start} + $${bytes_fat12_size}, bytes_fat12_start bytes_fat12_size,}}
# It is probably not necessary to align the GPT mirror end on a track boundary.
${eval ${call offset,bytes_gpt_mirror_size, $${sectors_gpt_mirror_size} * $${sector_size}, sectors_gpt_mirror_size,}}
${eval ${call offset,bytes_gpt_mirror_end, $${bytes_fat12_end} + $${bytes_gpt_mirror_size}, bytes_fat12_end bytes_gpt_mirror_size,}}
${eval ${call offset,bytes_gpt_mirror_start, $${bytes_gpt_mirror_end} - $${bytes_gpt_mirror_size}, bytes_gpt_mirror_end bytes_gpt_mirror_size,}}
${eval ${call offset,bytes_zip_end, $${bytes_os_size}, bytes_os_size,}}
os_fat12_partition = "$@@@${bytes_fat12_start}"
${bld}/os.fat12: ${bld}/os.zip ${dep_bytes_fat12_size} ${dep_bytes_fat12_start} ${dep_sectors_os_size} \
./utils/mformat ./utils/mcopy ${bld}/check_makefile
set -x; dd if=/dev/zero bs=${sector_size} count=${sectors_os_size} of=$@
./utils/faketime.sh ${commit_timestamp_iso_8601} ./utils/mformat -v "Example OS" \
-T ${sectors_fat12_size} \
-h ${os_floppy_chs_h} \
-s ${os_floppy_chs_s} \
-i ${os_fat12_partition}
./utils/faketime.sh ${commit_timestamp_iso_8601} ./utils/mcopy -i ${os_fat12_partition} $< "::os.zip"
${bld}/iso_files/os.zip: ${bld}/os.zip ${bld}/check_makefile
# TODO: make it so that the various file formats are mutual quines:
# * the ISO should contain the original file
# * the ZIP should contain the original file
# * the FAT12 should contain the original file
cp $< $@
# 4 sectors loaded when booting from optical media (CD-ROM, …):
${bld}/iso_files/boot/iso_boot.sys: ${bld}/os.32k ${bld}/check_makefile
# TODO: this copy of the (or alternate) bootsector should contain a Boot Information Table,
# see https://wiki.osdev.org/El-Torito#A_BareBones_Boot_Image_with_Boot_Information_Table
dd if=$< bs=512 count=4 of=$@
${bld}/os.zip: ${bld}/os.32k ${bld}/check_makefile
# We copy os.32k and alter its timestamp to ensure reproducible
# builds.
mkdir -p ${bld}/os.32k.tmp
cp -a $< ${bld}/os.32k.tmp/os.32k
touch -t ${commit_timestamp} ${bld}/os.32k.tmp/os.32k
(cd ${bld}/os.32k.tmp/ && zip -X ../os.zip os.32k)
rm ${bld}/os.32k.tmp/os.32k
rmdir ${bld}/os.32k.tmp
${bld}/os.zip.adjusted: ${bld}/os.zip ${dep_bytes_zip_start} ${bld}/check_makefile
# TODO: the ZIP file can end with a variable-length comment, this would allow us to hide the GPT mirrors.
set -x; dd if=/dev/zero bs=1 count=${bytes_zip_start} of=$@
cat $< >> $@
zip --adjust-sfx $@
gdisk_pipe_commands_slowly=while read str; do echo "$$str"; printf "\033[1;33m%s\033[m\n" "$$str" >&2; sleep 0.01; done
commit_hash_as_guid=$$(git log -1 --pretty=format:%H | sed -e 's/^\(.\{8\}\)\(.\{4\}\)\(.\{4\}\)\(.\{4\}\)\(.\{12\}\).*$$/\1-\2-\3-\4-\5/' | tr '[:lower:]' '[:upper:]')
git_dirty=test -n "$$(git diff --shortstat)"
gpt_disk_guid=${commit_hash_as_guid}$$(if $$git_dirty; then printf '0'; else printf '2'; fi)
gpt_partition_guid=${commit_hash_as_guid}$$(if $$git_dirty; then printf '1'; else printf '3'; fi)
${os_filename}: ${bld}/os.32k ${bld}/os.iso ${bld}/os.fat12 ${bld}/os.zip.adjusted \
${dep_bytes_header_32k_start} \
${dep_bytes_header_32k_size} \
${dep_bytes_fat12_start} \
${dep_bytes_fat12_size} \
${dep_bytes_gpt_mirror_start} \
${dep_bytes_gpt_mirror_end} \
${dep_sectors_fat12_start} \
${dep_sectors_fat12_size} \
${dep_bytes_zip_start} \
${bld}/check_makefile
rm -f $@
# start with the .iso
cp ${bld}/os.iso $@
# splice in the first 32k (bootsector and partition table)
set -x; dd skip=${bytes_header_32k_start} seek=${bytes_header_32k_start} bs=1 count=${bytes_header_32k_size} conv=notrunc if=${bld}/os.32k of=$@
# splice in fat12
set -x; dd skip=${bytes_fat12_start} seek=${bytes_fat12_start} bs=1 count=${bytes_fat12_size} conv=notrunc if=${bld}/os.fat12 of=$@
# pad with zeroes to prepare for GPT table
set -x; dd if=/dev/zero seek=$$((${bytes_gpt_mirror_end} - 1 )) bs=1 count=1 conv=notrunc of=$@
# patch the partition table
# Thanks to https://wiki.gentoo.org/wiki/Hybrid_partition_table for showing that gdisk can be used to make a hybrid MBR / GPT.
# gdisk commands:
# * Delete (the only) partition, eXpert mode, sector aLignment = 1, back to Main menu,
# * New partition (number = 1, start sector, end sector, type 0700)
# * Recovery and transformation options, make Hybrid,
# * add GPT partition #1 to the hybrid MBR, do Not put the EFI partition first,
# * MBR partition type=0x01, bootable=Yes, do Not add extra partitions,
# * back to Main menu,
# * eXpert mode,
# * Change partition GUID, the GUID itself,
# * change disk GUID, the GUID itself,
# * Print GPT, print prOtective MBR, Write, Proceed.
(if test "$$(uname -o)" = "Cygwin"; then printf "Y\n"; fi; \
printf "d\nx\nl\n1\nm\n"; \
printf "n\n1\n${sectors_fat12_start}\n${sectors_fat12_size}\n0700\n"; \
printf "r\nh\n"; \
printf "1\nN\n"; \
printf "01\nY\nN\n"; \
printf "x\n"; \
printf "g\n${gpt_disk_guid}\n"; \
printf "c\n${gpt_partition_guid}\n"; \
printf "p\no\nw\nY\n") | ${gdisk_pipe_commands_slowly} | gdisk $@
# Inject MS-DOS newlines (CR+LF) and comments (":: ") in the GUID field of unused partition table entries,
# so that the part that is to be skipped by MS-DOS does not form a line longer than the MS-DOS maximum
# line length (8192 excluding CR+LF). $i below is the partition entry number, starting from 1
# The numbers 55 and 118 are arbitrarily chosen so that the space between two CR+LF is less than 8192.
for i in 55 118; do \
printf "\r\n:: %02x" $$i | dd bs=1 seek=$$(( 1024 + ( ($$i) - 1) * 128 + 16)) count=7 conv=notrunc of=$@; \
printf "\r\n:: %02x" $$i | dd bs=1 seek=$$(( ${bytes_gpt_mirror_start} + ( ($$i) - 1) * 128 + 16)) count=7 conv=notrunc of=$@; \
done
# splice in zip at the end
set -x; dd skip=${bytes_zip_start} seek=${bytes_zip_start} bs=1 conv=notrunc if=${bld}/os.zip.adjusted of=$@
chmod a+x-w $@
${bld}/os.file: ${os_filename} ${bld}/check_makefile
file -kr $< > $@
${bld}/os.gdisk: ${os_filename} ${bld}/check_makefile
# gdisk commands:
# * eXpert mode
# * Print partition table
#  * print detailed Information about the (only) partition
# * print prOtective MBR table
# * Quit
printf '2\nx\np\ni\nr\no\nq\n' | ${gdisk_pipe_commands_slowly} | gdisk $< | tee $@
${bld}/os.offsets.hex: ${offset_names:%=${bld}/offsets/%.hex} ${bld}/check_makefile
grep '^' ${offset_names:%=${bld}/offsets/%.hex} | sed -e 's|^.*/||' -e 's/:/: 0x/' | column -t > $@
${bld}/os.offsets.dec: ${offset_names:%=${bld}/offsets/%.dec} ${bld}/check_makefile
grep '^' ${offset_names:%=${bld}/offsets/%.dec} | sed -e 's|^.*/||' -e 's/:/: /' | column -t > $@
${bld}/offsets/%.hex: ${bld}/offsets/%.dec
printf '%x\n' $$(cat $<) > $@
${bld}/os.hex_with_offsets: ${os_filename} ${bld}/os.offsets.hex
hexdump -C $< \
| grep -E -e "($$(cat ${bld}/os.offsets.hex | cut -d '=' -f 2 | sed -e 's/^[[:space:]]*0x\(.*\).$$/^\10/' | tr '\n' '|')^)" --color=yes > $@
${bld}/os.ndisasm.disasm: ${os_filename} utils/compact-ndisasm.sh ${bld}/check_makefile
./utils/compact-ndisasm.sh $< $@
${bld}/os.reasm.asm: ${bld}/os.ndisasm.disasm ${bld}/check_makefile
sed -e 's/^[^ ]\+ \+[^ ]\+ \+//' $< > $@

159
Makefile.test-example-os Normal file
View File

@ -0,0 +1,159 @@
built_files += ${bld}/checkerboard_800x600.xbm \
${bld}/checkerboard_1024x768.png \
${bld}/bochsrc \
${bld}/bochscontinue \
${bld}/twm_cfg \
${bld}/virtualbox.img \
${tests_emu:test/%=${bld}/test_pass/emu_%} \
${tests_noemu:test/%=${bld}/test_pass/noemu_%} \
${tests_requiring_sudo:test/%=${bld}/test_pass/sudo_%} \
${tests_emu:test/%=${screenshots}/%.png} \
${tests_emu:test/%=${screenshots}/%-anim.gif} \
utils/mformat utils/mcopy utils/mkisofs
built_directories += ${bld}/mnt_fat12 \
${bld}/mnt_iso \
${bld}/test_pass \
${screenshots}
${bld}/test_pass/noemu_%.reasm ${bld}/%.reasm: ${bld}/%.reasm.asm ${os_filename} utils/compact-ndisasm.sh ${bld}/check_makefile
# For now ignore this test, since we cannot have a reliable re-assembly of arbitrary data.
touch ${bld}/test_pass/noemu_$*.reasm ${bld}/$*.reasm
# nasm $< -o $@
# @echo "diff $@ ${os_filename}"
# @diff $@ ${os_filename} \
# && echo "Re-assembled file is identical to ${os_filename}" \
# || (./utils/compact-ndisasm.sh $@ ${bld}/os.reasm.disasm; \
# echo "Re-assembled file is different from ${os_filename}. Use meld ${bld}/os.ndisasm.disasm ${bld}/os.reasm.disasm to see differences."; \
# exit 0)
#os.arm.disasm: ${os_filename} ${bld}/check_makefile
# arm-none-eabi-objdump --endian=little -marm -b binary -D --adjust-vma=0x8000 $< > $@
.PHONY: test
test: ${tests_emu:test/%=${bld}/test_pass/emu_%} \
${tests_noemu:test/%=${bld}/test_pass/noemu_%} \
${tests_requiring_sudo:test/%=${bld}/test_pass/sudo_%} \
${bld}/check_makefile
.PHONY: test/emu test/noemu test/requiring_sudo
test/emu: ${tests_emu} ${bld}/check_makefile
test/requiring_sudo: ${tests_requiring_sudo} ${bld}/check_makefile
test/noemu: ${tests_noemu} ${bld}/check_makefile
.PHONY: ${tests_emu} ${tests_noemu} ${tests_requiring_sudo}
${tests_emu}: ${bld}/test_pass/emu_$$(@F) ${bld}/check_makefile
${tests_noemu}: ${bld}/test_pass/noemu_$$(@F) ${bld}/check_makefile
${tests_requiring_sudo}: ${bld}/test_pass/sudo_$$(@F) ${bld}/check_makefile
${bld}/test_pass/emu_% ${screenshots}/%.png ${screenshots}/%-anim.gif: \
${os_filename} \
${bld}/checkerboard_800x600.xbm \
utils/gui-wrapper.sh utils/ansi-screenshots/ansi_screenshot.sh utils/ansi-screenshots/to_ansi.sh \
test/%.sh \
${bld}/check_makefile \
| ${bld}/test_pass ${screenshots}
./utils/gui-wrapper.sh 800x600x24 ./test/$*.sh $<
touch ${bld}/test_pass/emu_$*
${bld}/test_pass/noemu_zip: ${os_filename} ${bld}/check_makefile
unzip -t ${os_filename}
touch $@
${bld}/test_pass/noemu_sizes: ${bld}/os.32k ${os_filename} ${bld}/check_makefile
test "$$(utils/file-length.sh -c ${bld}/os.32k)" = "$$((32*1024))"
test "$$(utils/file-length.sh -c ${os_filename})" = "$$((1440*1024))"
touch $@
# check that the fat filesystem has the correct contents
${bld}/test_pass/noemu_fat12_contents: ${os_filename} ${dep_bytes_fat12_start} ${bld}/check_makefile
mdir -i "$<@@${bytes_fat12_start}" :: | grep -E "^os[[:space:]]+zip[[:space:]]+"
touch $@
.PHONY: test/requiring_sudo
test/requiring_sudo: ${tests_requiring_sudo:test/%=${bld}/test_pass/sudo_%} ${bld}/check_makefile
# check that the fat filesystem can be mounted and has the correct contents
${bld}/test_pass/sudo_fat12_mount: ${os_filename} ${dep_bytes_fat12_start} ${bld}/check_makefile | ${bld}/mnt_fat12
sudo umount ${bld}/mnt_fat12 || true
# debug failure to mount the FAT12 filesystem
(set -x ;\
grep '^' ${bld}/offsets/* ;\
(sudo mount -o loop,ro,offset=${bytes_fat12_start} $< ${bld}/mnt_fat12) || true ;\
dmesg | tail ;\
ls -l ${bld}/mnt_fat12 | (grep os.zip || true) ;\
sudo umount ${bld}/mnt_fat12 || true ;\
sleep 10 ;\
cat ${os_filename} | xz | base64 ;\
(sudo mount -o loop,ro,offset=${bytes_fat12_start} $< ${bld}/mnt_fat12) || true ;\
sudo umount ${bld}/mnt_fat12 || true ;\
) > debug-fat12.log 2>&1
cat debug-fat12.log
sudo mount -o loop,ro,offset=${bytes_fat12_start} $< ${bld}/mnt_fat12
ls -l ${bld}/mnt_fat12 | grep os.zip
sudo umount ${bld}/mnt_fat12
touch $@
${bld}/test_pass/sudo_iso_mount: ${os_filename} ${bld}/check_makefile | ${bld}/mnt_iso
sudo umount ${bld}/mnt_iso || true
# debug failure to mount the ISO9660 filesystem
(set -x ;\
grep '^' ${bld}/offsets/* ;\
(sudo mount -o loop,ro $< ${bld}/mnt_iso) || true ;\
dmesg | tail ;\
cat ${os_filename} | xz | base64 ;\
ls -l ${bld}/mnt_iso | grep os.zip ;\
sudo umount ${bld}/mnt_iso || true ;\
sleep 11 ;\
(sudo mount -o loop,ro $< ${bld}/mnt_iso) || true ;\
sudo umount ${bld}/mnt_iso || true ;\
) > debug-iso.log 2>&1
cat debug-iso.log
sudo mount -o loop,ro $< ${bld}/mnt_iso
sudo umount ${bld}/mnt_iso
touch $@
.PHONY: test/macos
test/macos: all test/noemu test/macos-sh test/macos-sh-x11
.PHONY: test/macos-sh-x11
test/macos-sh-x11:
sudo mkdir -p /tmp/.X11-unix
sudo chmod a+rwxt /tmp/.X11-unix
xvfb :42 & \
sleep 5; \
DISPLAY=:42 xterm -e ./${os_filename} & \
sleep 5; \
# DISPLAY=:42 import -window root ${screenshots}/macos-sh-x11.png
screencapture ${screenshots}/macos-sh-x11-screencapture.png
.PHONY: test/macos-sh
test/macos-sh: ${bld}/check_makefile \
${bld}/checkerboard_1024x768.png \
| ${screenshots}
osascript -e 'tell app "Terminal" to do script "'"$$PWD"'/${os_filename}"'
sleep 2
osascript -e 'tell app "Terminal" to activate'
sleep 5
(date +%n && sleep 0.2 && date +%n) || true
screencapture ${screenshots}/screencapture-os-bat.png
./utils/gui-wrapper-mac.sh 1024x768x24 ./test/gui-sh-mac.sh ${os_filename}
# See https://wiki.osdev.org/EFI#Emulation to emulate an UEFI system with qemu, to test the EFI boot from hdd / cd / fd (?).
# Create checkerboard background
${bld}/checkerboard_%.png: ${bld}/check_makefile
convert -size "$*" \
tile:pattern:checkerboard \
-auto-level +level-colors 'gray(192),gray(128)' \
$@
${bld}/checkerboard_%.xbm: ${bld}/check_makefile
convert -size "$*" \
tile:pattern:checkerboard \
-auto-level \
$@
# Temporary files
${bld}/bochsrc ${bld}/bochscontinue ${bld}/twm_cfg ${bld}/virtualbox.img: ${bld}/check_makefile
touch $@

View File

@ -1,7 +1,5 @@
max_jobs: 1
# Are both necessary?
shallow_clone: true
clone_depth: 1
image: Visual Studio 2015
@ -24,7 +22,7 @@ install:
- cmd: cd ..
- cmd: set PATH=%cd%\gdisk\;%PATH%
# mkisofs,zip,ImageMagick
- cmd: C:\cygwin\setup-x86.exe --no-replaceonreboot --no-shortcuts --quiet-mode --root c:\cygwin --packages mkisofs,genisoimage,zip,ImageMagick,xorg-server-extra
- cmd: C:\cygwin\setup-x86.exe --no-replaceonreboot --no-shortcuts --quiet-mode --root c:\cygwin --packages mkisofs,genisoimage,zip,ImageMagick,xorg-server-extra,datefudge
- cmd: C:\cygwin\bin\find C:\cygwin -iname "import*"
- cmd: C:\cygwin\bin\find C:\cygwin -iname "convert*"
# mtools
@ -57,7 +55,7 @@ install:
- cmd: echo TODO install gnu make and dependencies
build_script:
- cmd: make all
- cmd: make COMMIT_TIMESTAMP_ISO_8601="%APPVEYOR_REPO_COMMIT_TIMESTAMP%" all
test_script:
- cmd: cmd /C os.bat exit

View File

@ -3,6 +3,7 @@ set -e
if test $# -ne 1 || test "$1" = '-h' -o "$1" = '--help'; then
echo "Usage: $0 operating_system_file"
exit 1
fi
os_filename="$1"

View File

@ -3,6 +3,7 @@ set -e
if test $# -ne 1 || test "$1" = '-h' -o "$1" = '--help'; then
echo "Usage: $0 operating_system_file"
exit 1
fi
os_filename="$1"

View File

@ -3,6 +3,7 @@ set -e
if test $# -ne 1 || test "$1" = '-h' -o "$1" = '--help'; then
echo "Usage: $0 operating_system_file"
exit 1
fi
# Force the path to be relative or absolute, but with at least one /
# Otherwise, the command will be searched in the $PATH, instead of using the

View File

@ -3,6 +3,7 @@ set -e
if test $# -ne 1 || test "$1" = '-h' -o "$1" = '--help'; then
echo "Usage: $0 operating_system_file"
exit 1
fi
# Force the path to be relative or absolute, but with at least one /
# Otherwise, the command will be searched in the $PATH, instead of using the

View File

@ -3,6 +3,7 @@ set -e
if test $# -ne 1 || test "$1" = '-h' -o "$1" = '--help'; then
echo "Usage: $0 operating_system_file"
exit 1
fi
os_filename="$1"

View File

@ -3,6 +3,7 @@ set -e
if test $# -ne 1 || test "$1" = '-h' -o "$1" = '--help'; then
echo "Usage: $0 operating_system_file"
exit 1
fi
os_filename="$1"

View File

@ -3,6 +3,7 @@ set -e
if test $# -ne 1 || test "$1" = '-h' -o "$1" = '--help'; then
echo "Usage: $0 operating_system_file"
exit 1
fi
os_filename="$1"

24
utils/faketime.sh Executable file
View File

@ -0,0 +1,24 @@
#!/bin/bash
set -euET -o pipefail
timestamp_iso_8601="$1"
shift
date_command() {
# TODO: substring or case … in Darwin*)
if test "$(uname -s)" = Darwin; then
date -j -f '%Y-%m-%dT%H:%M:%S' "$(echo "${1}" | cut -c 1-19)" "${2}";
else
date -d "${1}" "${2}";
fi
}
if which faketime >/dev/null; then
( set -x; faketime -f "$(date_command "${timestamp_iso_8601}" '+%Y-%m-%d %H:%m:%S')" "$@"; )
elif which datefudge >/dev/null; then
( set -x; datefudge --static "${timestamp_iso_8601}" "$@"; )
else
echo "ERROR: command faketime or datefudge not found. Please install either command."
exit 1
fi

View File

@ -5,5 +5,5 @@ set -e
# The sikuli-ide packaged with ubuntu 16.04 does not seem to work correctly: missing dependencies, some dependencies are too recent, …
mkdir ~/sikulix/
wget https://launchpad.net/sikuli/sikulix/1.1.3/+download/sikulixsetup-1.1.3.jar -O ~/sikulix/sikulixsetup-1.1.3.jar
(cd ~/sikulix && java -jar sikulixsetup-1.1.3.jar options 1 1.1)
(cd ~/sikulix && java -jar sikulixsetup-1.1.3.jar options 1 1.1) || (cat ~/sikulix/SikuliX-1.1.3-SetupLog.txt; exit 1)
echo 'export PATH="$HOME/sikulix/:$PATH"'