diff --git a/.gitmodules b/.gitmodules index 57a4e62df1..e69e19b82c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "stex"] path = stex url = https://github.com/dybvig/stex +[submodule "lz4"] + path = lz4 + url = https://github.com/lz4/lz4.git diff --git a/LOG b/LOG index 9276d1dd6d..6c542b6966 100644 --- a/LOG +++ b/LOG @@ -1204,3 +1204,10 @@ compile.ss - minor build and new-release updates checkin, newrelease, Makefile.in, Makefile-workarea.in +- change the default compression mode to LZ4 and add a compress-format + parameter to select a compression format for output; input infers the + compression format + io.ss, bytevector.ss, back.ss, primdata.ss, + compress.c (new), new-io.c, fasl.c, scheme.c, compress.h (new), + externs.h, system.h, expeditor.c, configure, Mf-*, Makefile.*nt, + workarea, mat.ss, io.ms, io.stex, objects.stex, release_notes.stex diff --git a/c/Makefile.a6nt b/c/Makefile.a6nt index 04c9659e71..6db91e54a8 100644 --- a/c/Makefile.a6nt +++ b/c/Makefile.a6nt @@ -28,7 +28,7 @@ MDMain = ..\boot\$m\mainmd.obj ResFile = ..\boot\$m\scheme.res # We use MD so that we can link with and load DLLs built against msvcrxxx.dll -CFLAGS=/nologo /Ox /W3 /Zi /I$(SchemeInclude) /I..\zlib /DUSE_ANSI_PROTOTYPES /DX86_64 /DWIN32 /D_CRT_SECURE_NO_WARNINGS +CFLAGS=/nologo /Ox /W3 /Zi /I$(SchemeInclude) /I..\zlib /I..\lz4\lib /DUSE_ANSI_PROTOTYPES /DX86_64 /DWIN32 /D_CRT_SECURE_NO_WARNINGS MDCFLAGS=$(CFLAGS) /MD MTCFLAGS=$(CFLAGS) /MT DLLLDFLAGS=/debug:full /machine:X64 /nologo @@ -37,27 +37,29 @@ DLLLDFLAGS=/debug:full /machine:X64 /nologo EXELDFLAGS=/debug:full /machine:X64 /incremental:no /nologo /STACK:0x1000000 # use following flags for debugging -# CFLAGS=/nologo /Od /W3 /Zi /I$(SchemeInclude) /I..\zlib /DUSE_ANSI_PROTOTYPES /DX86_64 /DWIN32 /D_CRT_SECURE_NO_WARNINGS +# CFLAGS=/nologo /Od /W3 /Zi /I$(SchemeInclude) /I..\zlib /I..\lz4\lib /DUSE_ANSI_PROTOTYPES /DX86_64 /DWIN32 /D_CRT_SECURE_NO_WARNINGS # MDCFLAGS=$(CFLAGS) /MDd # MTCFLAGS=$(CFLAGS) /MTd SystemLib=rpcrt4.lib ole32.lib advapi32.lib User32.lib MDZlibLib=..\zlib\zlib.lib MTZlibLib=..\zlib\zlibmt.lib +MDLZ4Lib=..\lz4\lib\liblz4.lib +MTLZ4Lib=..\lz4\lib\liblz4mt.lib csrc=statics.c segment.c alloc.c symbol.c intern.c gcwrapper.c gc-oce.c gc-ocd.c\ number.c schsig.c io.c new-io.c print.c fasl.c stats.c\ foreign.c prim.c prim5.c flushcache.c\ windows.c\ - schlib.c thread.c expeditor.c scheme.c + schlib.c thread.c expeditor.c scheme.c compress.c cobj=statics.obj segment.obj alloc.obj symbol.obj intern.obj gcwrapper.obj gc-oce.obj gc-ocd.obj\ number.obj schsig.obj io.obj new-io.obj print.obj fasl.obj stats.obj\ foreign.obj prim.obj prim5.obj flushcache.obj\ windows.obj\ - schlib.obj thread.obj expeditor.obj scheme.obj + schlib.obj thread.obj expeditor.obj scheme.obj compress.obj -hsrc=system.h types.h version.h globals.h externs.h segment.h gc.c thread.h sort.h itest.c +hsrc=system.h types.h version.h globals.h externs.h compress.h segment.h gc.c thread.h sort.h itest.c .SUFFIXES: @@ -66,13 +68,14 @@ all: $(Exec) $(MTKernelLib) $(MDKernelLib) $(MTMain) $(KernelLib) $(MTKernelLib) $(MDKernelLib): $(hsrc) $(KernelLib) $(MTKernelLib) $(MDKernelLib): $(SchemeInclude)/equates.h $(SchemeInclude)/scheme.h $(KernelLib) $(MTKernelLib) $(MDKernelLib): ..\zlib/zconf.h ..\zlib/zlib.h +$(KernelLib) $(MTKernelLib) $(MDKernelLib): ../lz4/lib/lz4.h ../lz4/lib/lz4frame.h -$(MTKernelLib): $(csrc) $(MTZlibLib) +$(MTKernelLib): $(csrc) $(MTZlibLib) $(MTLZ4Lib) -del /f $(MTKernelLib) cl /DSCHEME_STATIC /c $(MTCFLAGS) $(csrc) link /lib /nologo -out:$(MTKernelLib) $(cobj) $(MTZlibLib) -$(MDKernelLib): $(csrc) $(MDZlibLib) +$(MDKernelLib): $(csrc) $(MDZlibLib) $(MDLZ4Lib) -del /f $(MDKernelLib) cl /DSCHEME_STATIC /c $(MDCFLAGS) $(csrc) link /lib /nologo -out:$(MDKernelLib) $(cobj) $(MDZlibLib) @@ -86,11 +89,11 @@ $(KernelDll): $(KernelLib) # conflicts with other dlls. use 'depends ' to check. # we no longer attempt to rebase other the CRT dll since it # has already been signed. -$(KernelLib): $(ResFile) $(csrc) $(MDZlibLib) +$(KernelLib): $(ResFile) $(csrc) $(MDZlibLib) $(MDLZ4Lib) -del /f $(KernelLib) -del /f $(KernelDll) cl /c $(MDCFLAGS) $(csrc) - link -dll -out:$(KernelDll) $(DLLLDFLAGS) $(ResFile) $(cobj) $(MDZlibLib) $(SystemLib) + link -dll -out:$(KernelDll) $(DLLLDFLAGS) $(ResFile) $(cobj) $(MDZlibLib) $(MDLZ4Lib) $(SystemLib) editbin /nologo /rebase:base=0x67480000 $(KernelDll) $(MTMain): main.c @@ -130,6 +133,18 @@ mdscheme.exe: $(ResFile) $(MDMain) $(MDKernelLib) nmake /nologo -f win32/Makefile.msc AR="link /lib" cd ../c +$(MDLZ4Lib) $(MTLZ4Lib): ../lz4/lib/lz4.c ../lz4/lib/lz4frame.c ../lz4/lib/lz4hc.c + cl /c /Fo../lz4/lib/lz4.obj $(MDCFLAGS) ../lz4/lib/lz4.c + cl /c /Fo../lz4/lib/lz4frame.obj $(MDCFLAGS) ../lz4/lib/lz4frame.c + cl /c /Fo../lz4/lib/lz4hc.obj $(MDCFLAGS) ../lz4/lib/lz4hc.c + cl /c /Fo../lz4/lib/xxhash.obj $(MDCFLAGS) ../lz4/lib/xxhash.c + lib /OUT:$(MDLZ4Lib) ../lz4/lib/lz4.obj ../lz4/lib/lz4frame.obj ../lz4/lib/lz4hc.obj ../lz4/lib/xxhash.obj + cl /c /Fo../lz4/lib/lz4mt.obj $(MTCFLAGS) ../lz4/lib/lz4.c + cl /c /Fo../lz4/lib/lz4framemt.obj $(MTCFLAGS) ../lz4/lib/lz4frame.c + cl /c /Fo../lz4/lib/lz4hcmt.obj $(MTCFLAGS) ../lz4/lib/lz4hc.c + cl /c /Fo../lz4/lib/xxhashmt.obj $(MTCFLAGS) ../lz4/lib/xxhash.c + lib /OUT:$(MTLZ4Lib) ../lz4/lib/lz4mt.obj ../lz4/lib/lz4framemt.obj ../lz4/lib/lz4hcmt.obj ../lz4/lib/xxhashmt.obj + clean: -del /f $(cobj) main.obj $(KernelExp) -del /f mtscheme.exe diff --git a/c/Makefile.i3nt b/c/Makefile.i3nt index 279558721e..f080620556 100644 --- a/c/Makefile.i3nt +++ b/c/Makefile.i3nt @@ -28,7 +28,7 @@ MDMain = ..\boot\$m\mainmd.obj ResFile = ..\boot\$m\scheme.res # We use MD so that we can link with and load DLLs built against msvcrxxx.dll -CFLAGS=/nologo /fp:precise /Ox /W3 /Zi /I$(SchemeInclude) /I..\zlib /DUSE_ANSI_PROTOTYPES /DI386 /DWIN32 /D_CRT_SECURE_NO_WARNINGS +CFLAGS=/nologo /fp:precise /Ox /W3 /Zi /I$(SchemeInclude) /I..\zlib /I..\lz4\lib /DUSE_ANSI_PROTOTYPES /DI386 /DWIN32 /D_CRT_SECURE_NO_WARNINGS MDCFLAGS=$(CFLAGS) /MD MTCFLAGS=$(CFLAGS) /MT DLLLDFLAGS=/debug:full /machine:ix86 /nologo @@ -36,27 +36,29 @@ DLLLDFLAGS=/debug:full /machine:ix86 /nologo EXELDFLAGS=/debug:full /machine:ix86 /incremental:no /nologo /STACK:0x800000 # use following flags for debugging -# CFLAGS=/nologo /fp:precise /Od /W3 /Zi /I$(SchemeInclude) /I..\zlib /DUSE_ANSI_PROTOTYPES /DI386 /DWIN32 /D_CRT_SECURE_NO_WARNINGS +# CFLAGS=/nologo /fp:precise /Od /W3 /Zi /I$(SchemeInclude) /I..\zlib /I..\lz4\lib /DUSE_ANSI_PROTOTYPES /DI386 /DWIN32 /D_CRT_SECURE_NO_WARNINGS # MDCFLAGS=$(CFLAGS) /MDd # MTCFLAGS=$(CFLAGS) /MTd SystemLib=rpcrt4.lib ole32.lib advapi32.lib User32.lib MDZlibLib=..\zlib\zlib.lib MTZlibLib=..\zlib\zlibmt.lib +MDLZ4Lib=..\lz4\lib\liblz4.lib +MTLZ4Lib=..\lz4\lib\liblz4mt.lib csrc=statics.c segment.c alloc.c symbol.c intern.c gcwrapper.c gc-oce.c gc-ocd.c\ number.c schsig.c io.c new-io.c print.c fasl.c stats.c\ foreign.c prim.c prim5.c flushcache.c\ windows.c\ - schlib.c thread.c expeditor.c scheme.c + schlib.c thread.c expeditor.c scheme.c compress.c cobj=statics.obj segment.obj alloc.obj symbol.obj intern.obj gcwrapper.obj gc-oce.obj gc-ocd.obj\ number.obj schsig.obj io.obj new-io.obj print.obj fasl.obj stats.obj\ foreign.obj prim.obj prim5.obj flushcache.obj\ windows.obj\ - schlib.obj thread.obj expeditor.obj scheme.obj + schlib.obj thread.obj expeditor.obj scheme.obj compress.obj -hsrc=system.h types.h version.h globals.h externs.h segment.h gc.c thread.h sort.h itest.c +hsrc=system.h types.h version.h globals.h externs.h compress.h segment.h gc.c thread.h sort.h itest.c .SUFFIXES: @@ -65,13 +67,14 @@ all: $(Exec) $(MTKernelLib) $(MDKernelLib) $(MTMain) $(KernelLib) $(MTKernelLib) $(MDKernelLib): $(hsrc) $(KernelLib) $(MTKernelLib) $(MDKernelLib): $(SchemeInclude)/equates.h $(SchemeInclude)/scheme.h $(KernelLib) $(MTKernelLib) $(MDKernelLib): ..\zlib/zconf.h ..\zlib/zlib.h +$(KernelLib) $(MTKernelLib) $(MDKernelLib): ../lz4/lib/lz4.h ../lz4/lib/lz4frame.h -$(MTKernelLib): $(csrc) $(MTZlibLib) +$(MTKernelLib): $(csrc) $(MTZlibLib) $(MTLZ4Lib) -del /f $(MTKernelLib) cl /DSCHEME_STATIC /c $(MTCFLAGS) $(csrc) link /lib /nologo -out:$(MTKernelLib) $(cobj) $(MTZlibLib) -$(MDKernelLib): $(csrc) $(MDZlibLib) +$(MDKernelLib): $(csrc) $(MDZlibLib) $(MDLZ4Lib) -del /f $(MDKernelLib) cl /DSCHEME_STATIC /c $(MDCFLAGS) $(csrc) link /lib /nologo -out:$(MDKernelLib) $(cobj) $(MDZlibLib) @@ -85,11 +88,11 @@ $(KernelDll): $(KernelLib) # conflicts with other dlls. use 'depends ' to check. # we no longer attempt to rebase other the CRT dll since it # has already been signed. -$(KernelLib): $(ResFile) $(csrc) $(MDZlibLib) +$(KernelLib): $(ResFile) $(csrc) $(MDZlibLib) $(MDLZ4Lib) -del /f $(KernelLib) -del /f $(KernelDll) cl /c $(MDCFLAGS) $(csrc) - link -dll -out:$(KernelDll) $(DLLLDFLAGS) $(ResFile) $(cobj) $(MDZlibLib) $(SystemLib) + link -dll -out:$(KernelDll) $(DLLLDFLAGS) $(ResFile) $(cobj) $(MDZlibLib) $(MDLZ4Lib) $(SystemLib) editbin /nologo /rebase:base=0x67480000 $(KernelDll) $(MTMain): main.c @@ -129,6 +132,18 @@ mdscheme.exe: $(ResFile) $(MDMain) $(MDKernelLib) nmake /nologo -f win32/Makefile.msc AR="link /lib" cd ../c +$(MDLZ4Lib) $(MTLZ4Lib): ../lz4/lib/lz4.c ../lz4/lib/lz4frame.c ../lz4/lib/lz4hc.c + cl /c /Fo../lz4/lib/lz4.obj $(MDCFLAGS) ../lz4/lib/lz4.c + cl /c /Fo../lz4/lib/lz4frame.obj $(MDCFLAGS) ../lz4/lib/lz4frame.c + cl /c /Fo../lz4/lib/lz4hc.obj $(MDCFLAGS) ../lz4/lib/lz4hc.c + cl /c /Fo../lz4/lib/xxhash.obj $(MDCFLAGS) ../lz4/lib/xxhash.c + lib /OUT:$(MDLZ4Lib) ../lz4/lib/lz4.obj ../lz4/lib/lz4frame.obj ../lz4/lib/lz4hc.obj ../lz4/lib/xxhash.obj + cl /c /Fo../lz4/lib/lz4mt.obj $(MTCFLAGS) ../lz4/lib/lz4.c + cl /c /Fo../lz4/lib/lz4framemt.obj $(MTCFLAGS) ../lz4/lib/lz4frame.c + cl /c /Fo../lz4/lib/lz4hcmt.obj $(MTCFLAGS) ../lz4/lib/lz4hc.c + cl /c /Fo../lz4/lib/xxhashmt.obj $(MTCFLAGS) ../lz4/lib/xxhash.c + lib /OUT:$(MTLZ4Lib) ../lz4/lib/lz4mt.obj ../lz4/lib/lz4framemt.obj ../lz4/lib/lz4hcmt.obj ../lz4/lib/xxhashmt.obj + clean: -del /f $(cobj) main.obj $(KernelExp) -del /f mtscheme.exe diff --git a/c/Makefile.ta6nt b/c/Makefile.ta6nt index 637f1b4889..a446b09c2d 100644 --- a/c/Makefile.ta6nt +++ b/c/Makefile.ta6nt @@ -28,7 +28,7 @@ MDMain = ..\boot\$m\mainmd.obj ResFile = ..\boot\$m\scheme.res # We use MD so that we can link with and load DLLs built against msvcrxxx.dll -CFLAGS=/nologo /Ox /W3 /Zi /I$(SchemeInclude) /I..\zlib /DUSE_ANSI_PROTOTYPES /DX86_64 /DWIN32 /D_CRT_SECURE_NO_WARNINGS +CFLAGS=/nologo /Ox /W3 /Zi /I$(SchemeInclude) /I..\zlib /I..\lz4\lib /DUSE_ANSI_PROTOTYPES /DX86_64 /DWIN32 /D_CRT_SECURE_NO_WARNINGS MDCFLAGS=$(CFLAGS) /MD MTCFLAGS=$(CFLAGS) /MT DLLLDFLAGS=/debug:full /machine:X64 /nologo @@ -37,27 +37,29 @@ DLLLDFLAGS=/debug:full /machine:X64 /nologo EXELDFLAGS=/debug:full /machine:X64 /incremental:no /nologo /STACK:0x1000000 # use following flags for debugging -# CFLAGS=/nologo /Od /W3 /Zi /I$(SchemeInclude) /I..\zlib /DUSE_ANSI_PROTOTYPES /DX86_64 /DWIN32 /D_CRT_SECURE_NO_WARNINGS +# CFLAGS=/nologo /Od /W3 /Zi /I$(SchemeInclude) /I..\zlib /I..\lz4\lib /DUSE_ANSI_PROTOTYPES /DX86_64 /DWIN32 /D_CRT_SECURE_NO_WARNINGS # MDCFLAGS=$(CFLAGS) /MDd # MTCFLAGS=$(CFLAGS) /MTd SystemLib=rpcrt4.lib ole32.lib advapi32.lib User32.lib MDZlibLib=..\zlib\zlib.lib MTZlibLib=..\zlib\zlibmt.lib +MDLZ4Lib=..\lz4\lib\liblz4.lib +MTLZ4Lib=..\lz4\lib\liblz4mt.lib csrc=statics.c segment.c alloc.c symbol.c intern.c gcwrapper.c gc-oce.c gc-ocd.c\ number.c schsig.c io.c new-io.c print.c fasl.c stats.c\ foreign.c prim.c prim5.c flushcache.c\ windows.c\ - schlib.c thread.c expeditor.c scheme.c + schlib.c thread.c expeditor.c scheme.c compress.c cobj=statics.obj segment.obj alloc.obj symbol.obj intern.obj gcwrapper.obj gc-oce.obj gc-ocd.obj\ number.obj schsig.obj io.obj new-io.obj print.obj fasl.obj stats.obj\ foreign.obj prim.obj prim5.obj flushcache.obj\ windows.obj\ - schlib.obj thread.obj expeditor.obj scheme.obj + schlib.obj thread.obj expeditor.obj scheme.obj compress.obj -hsrc=system.h types.h version.h globals.h externs.h segment.h gc.c thread.h sort.h itest.c +hsrc=system.h types.h version.h globals.h externs.h compress.h segment.h gc.c thread.h sort.h itest.c .SUFFIXES: @@ -66,13 +68,14 @@ all: $(Exec) $(MTKernelLib) $(MDKernelLib) $(MTMain) $(KernelLib) $(MTKernelLib) $(MDKernelLib): $(hsrc) $(KernelLib) $(MTKernelLib) $(MDKernelLib): $(SchemeInclude)/equates.h $(SchemeInclude)/scheme.h $(KernelLib) $(MTKernelLib) $(MDKernelLib): ..\zlib/zconf.h ..\zlib/zlib.h +$(KernelLib) $(MTKernelLib) $(MDKernelLib): ../lz4/lib/lz4.h ../lz4/lib/lz4frame.h -$(MTKernelLib): $(csrc) $(MTZlibLib) +$(MTKernelLib): $(csrc) $(MTZlibLib) $(MTLZ4Lib) -del /f $(MTKernelLib) cl /DSCHEME_STATIC /c $(MTCFLAGS) $(csrc) link /lib /nologo -out:$(MTKernelLib) $(cobj) $(MTZlibLib) -$(MDKernelLib): $(csrc) $(MDZlibLib) +$(MDKernelLib): $(csrc) $(MDZlibLib) $(MDLZ4Lib) -del /f $(MDKernelLib) cl /DSCHEME_STATIC /c $(MDCFLAGS) $(csrc) link /lib /nologo -out:$(MDKernelLib) $(cobj) $(MDZlibLib) @@ -86,11 +89,11 @@ $(KernelDll): $(KernelLib) # conflicts with other dlls. use 'depends ' to check. # we no longer attempt to rebase other the CRT dll since it # has already been signed. -$(KernelLib): $(ResFile) $(csrc) $(MDZlibLib) +$(KernelLib): $(ResFile) $(csrc) $(MDZlibLib) $(MDLZ4Lib) -del /f $(KernelLib) -del /f $(KernelDll) cl /c $(MDCFLAGS) $(csrc) - link -dll -out:$(KernelDll) $(DLLLDFLAGS) $(ResFile) $(cobj) $(MDZlibLib) $(SystemLib) + link -dll -out:$(KernelDll) $(DLLLDFLAGS) $(ResFile) $(cobj) $(MDZlibLib) $(MDLZ4Lib) $(SystemLib) editbin /nologo /rebase:base=0x67480000 $(KernelDll) $(MTMain): main.c @@ -130,6 +133,18 @@ mdscheme.exe: $(ResFile) $(MDMain) $(MDKernelLib) nmake /nologo -f win32/Makefile.msc AR="link /lib" cd ../c +$(MDLZ4Lib) $(MTLZ4Lib): ../lz4/lib/lz4.c ../lz4/lib/lz4frame.c ../lz4/lib/lz4hc.c + cl /c /Fo../lz4/lib/lz4.obj $(MDCFLAGS) ../lz4/lib/lz4.c + cl /c /Fo../lz4/lib/lz4frame.obj $(MDCFLAGS) ../lz4/lib/lz4frame.c + cl /c /Fo../lz4/lib/lz4hc.obj $(MDCFLAGS) ../lz4/lib/lz4hc.c + cl /c /Fo../lz4/lib/xxhash.obj $(MDCFLAGS) ../lz4/lib/xxhash.c + lib /OUT:$(MDLZ4Lib) ../lz4/lib/lz4.obj ../lz4/lib/lz4frame.obj ../lz4/lib/lz4hc.obj ../lz4/lib/xxhash.obj + cl /c /Fo../lz4/lib/lz4mt.obj $(MTCFLAGS) ../lz4/lib/lz4.c + cl /c /Fo../lz4/lib/lz4framemt.obj $(MTCFLAGS) ../lz4/lib/lz4frame.c + cl /c /Fo../lz4/lib/lz4hcmt.obj $(MTCFLAGS) ../lz4/lib/lz4hc.c + cl /c /Fo../lz4/lib/xxhashmt.obj $(MTCFLAGS) ../lz4/lib/xxhash.c + lib /OUT:$(MTLZ4Lib) ../lz4/lib/lz4mt.obj ../lz4/lib/lz4framemt.obj ../lz4/lib/lz4hcmt.obj ../lz4/lib/xxhashmt.obj + clean: -del /f $(cobj) main.obj $(KernelExp) -del /f mtscheme.exe diff --git a/c/Makefile.ti3nt b/c/Makefile.ti3nt index 8dfa80e6fa..6803247908 100644 --- a/c/Makefile.ti3nt +++ b/c/Makefile.ti3nt @@ -28,7 +28,7 @@ MDMain = ..\boot\$m\mainmd.obj ResFile = ..\boot\$m\scheme.res # We use MD so that we can link with and load DLLs built against msvcrxxx.dll -CFLAGS=/nologo /fp:precise /Ox /W3 /Zi /I$(SchemeInclude) /I..\zlib /DUSE_ANSI_PROTOTYPES /DI386 /DWIN32 /D_CRT_SECURE_NO_WARNINGS +CFLAGS=/nologo /fp:precise /Ox /W3 /Zi /I$(SchemeInclude) /I..\zlib /I..\lz4\lib /DUSE_ANSI_PROTOTYPES /DI386 /DWIN32 /D_CRT_SECURE_NO_WARNINGS MDCFLAGS=$(CFLAGS) /MD MTCFLAGS=$(CFLAGS) /MT DLLLDFLAGS=/debug:full /machine:ix86 /nologo @@ -36,27 +36,29 @@ DLLLDFLAGS=/debug:full /machine:ix86 /nologo EXELDFLAGS=/debug:full /machine:ix86 /incremental:no /nologo /STACK:0x800000 # use following flags for debugging -# CFLAGS=/nologo /fp:precise /Od /W3 /Zi /I$(SchemeInclude) /I..\zlib /DUSE_ANSI_PROTOTYPES /DI386 /DWIN32 /D_CRT_SECURE_NO_WARNINGS +# CFLAGS=/nologo /fp:precise /Od /W3 /Zi /I$(SchemeInclude) /I..\zlib /I..\lz4\lib /DUSE_ANSI_PROTOTYPES /DI386 /DWIN32 /D_CRT_SECURE_NO_WARNINGS # MDCFLAGS=$(CFLAGS) /MDd # MTCFLAGS=$(CFLAGS) /MTd SystemLib=rpcrt4.lib ole32.lib advapi32.lib User32.lib MDZlibLib=..\zlib\zlib.lib MTZlibLib=..\zlib\zlibmt.lib +MDLZ4Lib=..\lz4\lib\liblz4.lib +MTLZ4Lib=..\lz4\lib\liblz4mt.lib csrc=statics.c segment.c alloc.c symbol.c intern.c gcwrapper.c gc-oce.c gc-ocd.c\ number.c schsig.c io.c new-io.c print.c fasl.c stats.c\ foreign.c prim.c prim5.c flushcache.c\ windows.c\ - schlib.c thread.c expeditor.c scheme.c + schlib.c thread.c expeditor.c scheme.c compress.c cobj=statics.obj segment.obj alloc.obj symbol.obj intern.obj gcwrapper.obj gc-oce.obj gc-ocd.obj\ number.obj schsig.obj io.obj new-io.obj print.obj fasl.obj stats.obj\ foreign.obj prim.obj prim5.obj flushcache.obj\ windows.obj\ - schlib.obj thread.obj expeditor.obj scheme.obj + schlib.obj thread.obj expeditor.obj scheme.obj compress.obj -hsrc=system.h types.h version.h globals.h externs.h segment.h gc.c thread.h sort.h itest.c +hsrc=system.h types.h version.h globals.h externs.h compress.h segment.h gc.c thread.h sort.h itest.c .SUFFIXES: @@ -65,13 +67,14 @@ all: $(Exec) $(MTKernelLib) $(MDKernelLib) $(MTMain) $(KernelLib) $(MTKernelLib) $(MDKernelLib): $(hsrc) $(KernelLib) $(MTKernelLib) $(MDKernelLib): $(SchemeInclude)/equates.h $(SchemeInclude)/scheme.h $(KernelLib) $(MTKernelLib) $(MDKernelLib): ..\zlib/zconf.h ..\zlib/zlib.h +$(KernelLib) $(MTKernelLib) $(MDKernelLib): ../lz4/lib/lz4.h ../lz4/lib/lz4frame.h -$(MTKernelLib): $(csrc) $(MTZlibLib) +$(MTKernelLib): $(csrc) $(MTZlibLib) $(MTLZ4Lib) -del /f $(MTKernelLib) cl /DSCHEME_STATIC /c $(MTCFLAGS) $(csrc) link /lib /nologo -out:$(MTKernelLib) $(cobj) $(MTZlibLib) -$(MDKernelLib): $(csrc) $(MDZlibLib) +$(MDKernelLib): $(csrc) $(MDZlibLib) $(MDLZ4Lib) -del /f $(MDKernelLib) cl /DSCHEME_STATIC /c $(MDCFLAGS) $(csrc) link /lib /nologo -out:$(MDKernelLib) $(cobj) $(MDZlibLib) @@ -85,11 +88,11 @@ $(KernelDll): $(KernelLib) # conflicts with other dlls. use 'depends ' to check. # we no longer attempt to rebase other the CRT dll since it # has already been signed. -$(KernelLib): $(ResFile) $(csrc) $(MDZlibLib) +$(KernelLib): $(ResFile) $(csrc) $(MDZlibLib) $(MDLZ4Lib) -del /f $(KernelLib) -del /f $(KernelDll) cl /c $(MDCFLAGS) $(csrc) - link -dll -out:$(KernelDll) $(DLLLDFLAGS) $(ResFile) $(cobj) $(MDZlibLib) $(SystemLib) + link -dll -out:$(KernelDll) $(DLLLDFLAGS) $(ResFile) $(cobj) $(MDZlibLib) $(MDLZ4Lib) $(SystemLib) editbin /nologo /rebase:base=0x67480000 $(KernelDll) $(MTMain): main.c @@ -129,6 +132,18 @@ mdscheme.exe: $(ResFile) $(MDMain) $(MDKernelLib) nmake /nologo -f win32/Makefile.msc AR="link /lib" cd ../c +$(MDLZ4Lib) $(MTLZ4Lib): ../lz4/lib/lz4.c ../lz4/lib/lz4frame.c ../lz4/lib/lz4hc.c + cl /c /Fo../lz4/lib/lz4.obj $(MDCFLAGS) ../lz4/lib/lz4.c + cl /c /Fo../lz4/lib/lz4frame.obj $(MDCFLAGS) ../lz4/lib/lz4frame.c + cl /c /Fo../lz4/lib/lz4hc.obj $(MDCFLAGS) ../lz4/lib/lz4hc.c + cl /c /Fo../lz4/lib/xxhash.obj $(MDCFLAGS) ../lz4/lib/xxhash.c + lib /OUT:$(MDLZ4Lib) ../lz4/lib/lz4.obj ../lz4/lib/lz4frame.obj ../lz4/lib/lz4hc.obj ../lz4/lib/xxhash.obj + cl /c /Fo../lz4/lib/lz4mt.obj $(MTCFLAGS) ../lz4/lib/lz4.c + cl /c /Fo../lz4/lib/lz4framemt.obj $(MTCFLAGS) ../lz4/lib/lz4frame.c + cl /c /Fo../lz4/lib/lz4hcmt.obj $(MTCFLAGS) ../lz4/lib/lz4hc.c + cl /c /Fo../lz4/lib/xxhashmt.obj $(MTCFLAGS) ../lz4/lib/xxhash.c + lib /OUT:$(MTLZ4Lib) ../lz4/lib/lz4mt.obj ../lz4/lib/lz4framemt.obj ../lz4/lib/lz4hcmt.obj ../lz4/lib/xxhashmt.obj + clean: -del /f $(cobj) main.obj $(KernelExp) -del /f mtscheme.exe diff --git a/c/Mf-a6fb b/c/Mf-a6fb index 9a55d7ae89..92bffb6814 100644 --- a/c/Mf-a6fb +++ b/c/Mf-a6fb @@ -27,15 +27,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib ${mdinclude} $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} ${mdinclude} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m64 ./configure --64) + (cd ../zlib; CFLAGS="${CFLAGS} -m64" ./configure --64) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m64" ${MAKE} liblz4.a) diff --git a/c/Mf-a6le b/c/Mf-a6le index 49b8d046c7..1959da5af5 100644 --- a/c/Mf-a6le +++ b/c/Mf-a6le @@ -26,15 +26,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -melf_x86_64 -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -melf_x86_64 -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m64 ./configure --64) + (cd ../zlib; CFLAGS="${CFLAGS} -m64" ./configure --64) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m64" ${MAKE} liblz4.a) diff --git a/c/Mf-a6nb b/c/Mf-a6nb index f3869b360e..5cf2ec043f 100644 --- a/c/Mf-a6nb +++ b/c/Mf-a6nb @@ -27,15 +27,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib ${mdinclude} $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} ${mdinclude} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m64 ./configure --64) + (cd ../zlib; CFLAGS="${CFLAGS} -m64" ./configure --64) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m64" ${MAKE} liblz4.a) diff --git a/c/Mf-a6ob b/c/Mf-a6ob index 03ab059065..12fa15eab1 100644 --- a/c/Mf-a6ob +++ b/c/Mf-a6ob @@ -27,15 +27,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib ${mdinclude} $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} ${mdinclude} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -Wl,--export-dynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m64 ./configure --64) + (cd ../zlib; CFLAGS="${CFLAGS} -m64" ./configure --64) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m64" ${MAKE} liblz4.a) diff --git a/c/Mf-a6osx b/c/Mf-a6osx index 8293937baa..a776bd3fd0 100644 --- a/c/Mf-a6osx +++ b/c/Mf-a6osx @@ -26,15 +26,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m64 ./configure --64) + (cd ../zlib; CFLAGS="${CFLAGS} -m64" ./configure --64) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m64" ${MAKE} liblz4.a) diff --git a/c/Mf-a6s2 b/c/Mf-a6s2 index 6d172cc73b..e3e6a1b8bf 100644 --- a/c/Mf-a6s2 +++ b/c/Mf-a6s2 @@ -26,15 +26,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -DSOLARIS -D${Cpu} -I${Include} -I../zlib $*.c + $C -c -DSOLARIS -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -melf_x86_64 -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -melf_x86_64 -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m64 ./configure --64) + (cd ../zlib; CFLAGS="${CFLAGS} -m64" ./configure --64) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m64" ${MAKE} liblz4.a) diff --git a/c/Mf-arm32le b/c/Mf-arm32le index a5bb283283..7cc8a6e620 100644 --- a/c/Mf-arm32le +++ b/c/Mf-arm32le @@ -26,15 +26,18 @@ mdobj = arm32le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: (cd ../zlib; ./configure) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; ${MAKE} liblz4.a) diff --git a/c/Mf-base b/c/Mf-base index cc23047ba7..c94f85d4cf 100644 --- a/c/Mf-base +++ b/c/Mf-base @@ -24,11 +24,11 @@ Scheme=../bin/$m/scheme kernelsrc=statics.c segment.c alloc.c symbol.c intern.c gcwrapper.c gc-ocd.c gc-oce.c\ number.c schsig.c io.c new-io.c print.c fasl.c stats.c foreign.c prim.c prim5.c flushcache.c\ - schlib.c thread.c expeditor.c scheme.c + schlib.c thread.c expeditor.c scheme.c compress.c kernelobj=${kernelsrc:%.c=%.$o} ${mdobj} -kernelhdr=system.h types.h version.h globals.h externs.h segment.h gc.c sort.h thread.h config.h itest.c +kernelhdr=system.h types.h version.h globals.h externs.h segment.h gc.c sort.h thread.h config.h compress.h itest.c mainsrc=main.c @@ -51,10 +51,10 @@ endif scheme.o: itest.c scheme.o main.o: config.h -${kernelobj}: system.h types.h version.h externs.h globals.h segment.h thread.h sort.h +${kernelobj}: system.h types.h version.h externs.h globals.h segment.h thread.h sort.h compress.h ${kernelobj}: ${Include}/equates.h ${Include}/scheme.h ${mainobj}: ${Include}/scheme.h -${kernelobj}: ../zlib/zconf.h ../zlib/zlib.h +${kernelobj}: ${zlibHeaderDep} ${LZ4HeaderDep} gc-ocd.o gc-oce.o: gc.c ../zlib/zlib.h ../zlib/zconf.h: ../zlib/configure.log @@ -62,6 +62,10 @@ gc-ocd.o gc-oce.o: gc.c ../zlib/libz.a: ../zlib/configure.log (cd ../zlib; ${MAKE}) +LZ4Sources=../lz4/lib/lz4.h ../lz4/lib/lz4frame.h \ + ../lz4/lib/lz4.c ../lz4/lib/lz4frame.c \ + ../lz4/lib/lz4hc.c ../lz4/lib/xxhash.c + clean: rm -f *.$o ${mdclean} rm -f Make.out diff --git a/c/Mf-i3fb b/c/Mf-i3fb index 834d8ab846..32f7042bf4 100644 --- a/c/Mf-i3fb +++ b/c/Mf-i3fb @@ -27,15 +27,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib ${mdinclude} $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} ${mdinclude} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m32 ./configure) + (cd ../zlib; CFLAGS="${CFLAGS} -m32" ./configure) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m32" ${MAKE} liblz4.a) diff --git a/c/Mf-i3le b/c/Mf-i3le index e9cf27d766..5bf9ab44b3 100644 --- a/c/Mf-i3le +++ b/c/Mf-i3le @@ -26,15 +26,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -melf_i386 -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -melf_i386 -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m32 ./configure) + (cd ../zlib; CFLAGS="${CFLAGS} -m32" ./configure) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m32" ${MAKE} liblz4.a) diff --git a/c/Mf-i3nb b/c/Mf-i3nb index f7378865c1..f0cfca976d 100644 --- a/c/Mf-i3nb +++ b/c/Mf-i3nb @@ -27,15 +27,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib ${mdinclude} $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} ${mdinclude} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m32 ./configure) + (cd ../zlib; CFLAGS="${CFLAGS} -m32" ./configure) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m32" ${MAKE} liblz4.a) diff --git a/c/Mf-i3ob b/c/Mf-i3ob index ee9ceec21b..237a7b3126 100644 --- a/c/Mf-i3ob +++ b/c/Mf-i3ob @@ -27,15 +27,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib ${mdinclude} $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} ${mdinclude} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -Wl,--export-dynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m32 ./configure) + (cd ../zlib; CFLAGS="${CFLAGS} -m32" ./configure) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m32" ${MAKE} liblz4.a) diff --git a/c/Mf-i3osx b/c/Mf-i3osx index 8ca333d76d..2c5f4b8ca7 100644 --- a/c/Mf-i3osx +++ b/c/Mf-i3osx @@ -26,15 +26,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m32 ./configure) + (cd ../zlib; CFLAGS="${CFLAGS} -m32" ./configure) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m32" ${MAKE} liblz4.a) diff --git a/c/Mf-i3qnx b/c/Mf-i3qnx index 25c24fd5ce..9c159d34a8 100644 --- a/c/Mf-i3qnx +++ b/c/Mf-i3qnx @@ -27,15 +27,18 @@ LocalInclude = /usr/local/include .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib -I${LocalInclude} $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} -I${LocalInclude} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -mi386nto -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -mi386nto -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -Wl,--export-dynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m32 ./configure) + (cd ../zlib; CFLAGS="${CFLAGS} -m32" ./configure) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m32" ${MAKE} liblz4.a) diff --git a/c/Mf-i3s2 b/c/Mf-i3s2 index 7f2c52963d..5be12fae4d 100644 --- a/c/Mf-i3s2 +++ b/c/Mf-i3s2 @@ -26,15 +26,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -DSOLARIS -D${Cpu} -I${Include} -I../zlib $*.c + $C -c -DSOLARIS -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -melf_i386 -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -melf_i386 -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m32 ./configure) + (cd ../zlib; CFLAGS="${CFLAGS} -m32" ./configure) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m32" ${MAKE} liblz4.a) diff --git a/c/Mf-ppc32le b/c/Mf-ppc32le index 844be15128..91705e772d 100644 --- a/c/Mf-ppc32le +++ b/c/Mf-ppc32le @@ -26,15 +26,18 @@ mdobj = ppc32.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m32 ./configure) + (cd ../zlib; CFLAGS="${CFLAGS} -m32" ./configure) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m32" ${MAKE} liblz4.a) diff --git a/c/Mf-ta6fb b/c/Mf-ta6fb index 81800dcacc..42a1dbb0fb 100644 --- a/c/Mf-ta6fb +++ b/c/Mf-ta6fb @@ -27,15 +27,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib ${mdinclude} $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} ${mdinclude} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m64 ./configure --64) + (cd ../zlib; CFLAGS="${CFLAGS} -m64" ./configure --64) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m64" ${MAKE} liblz4.a) diff --git a/c/Mf-ta6le b/c/Mf-ta6le index 30cc11d629..ee961471d6 100644 --- a/c/Mf-ta6le +++ b/c/Mf-ta6le @@ -26,15 +26,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -melf_x86_64 -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -melf_x86_64 -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m64 ./configure --64) + (cd ../zlib; CFLAGS="${CFLAGS} -m64" ./configure --64) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m64" ${MAKE} liblz4.a) diff --git a/c/Mf-ta6nb b/c/Mf-ta6nb index 6dbd83c74a..168c97103e 100644 --- a/c/Mf-ta6nb +++ b/c/Mf-ta6nb @@ -27,15 +27,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib ${mdinclude} $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} ${mdinclude} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m64 ./configure --64) + (cd ../zlib; CFLAGS="${CFLAGS} -m64" ./configure --64) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m64" ${MAKE} liblz4.a) diff --git a/c/Mf-ta6ob b/c/Mf-ta6ob index 45e29e8955..ef00f655ab 100644 --- a/c/Mf-ta6ob +++ b/c/Mf-ta6ob @@ -27,15 +27,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib ${mdinclude} $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} ${mdinclude} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -Wl,--export-dynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m64 ./configure --64) + (cd ../zlib; CFLAGS="${CFLAGS} -m64" ./configure --64) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m64" ${MAKE} liblz4.a) diff --git a/c/Mf-ta6osx b/c/Mf-ta6osx index 9626f36489..d070f3b3d0 100644 --- a/c/Mf-ta6osx +++ b/c/Mf-ta6osx @@ -26,15 +26,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m64 ./configure --64) + (cd ../zlib; CFLAGS="${CFLAGS} -m64" ./configure --64) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m64" ${MAKE} liblz4.a) diff --git a/c/Mf-ta6s2 b/c/Mf-ta6s2 index ed260d054c..2e54211373 100644 --- a/c/Mf-ta6s2 +++ b/c/Mf-ta6s2 @@ -26,15 +26,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -DSOLARIS -D${Cpu} -I${Include} -I../zlib $*.c + $C -c -DSOLARIS -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -melf_x86_64 -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -melf_x86_64 -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m64 ./configure --64) + (cd ../zlib; CFLAGS="${CFLAGS} -m64" ./configure --64) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m64" ${MAKE} liblz4.a) diff --git a/c/Mf-ti3fb b/c/Mf-ti3fb index 58af09cfcd..ec52090f8e 100644 --- a/c/Mf-ti3fb +++ b/c/Mf-ti3fb @@ -27,15 +27,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib ${mdinclude} $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} ${mdinclude} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m32 ./configure) + (cd ../zlib; CFLAGS="${CFLAGS} -m32" ./configure) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m32" ${MAKE} liblz4.a) diff --git a/c/Mf-ti3le b/c/Mf-ti3le index dad6af7c91..4d52e111e1 100644 --- a/c/Mf-ti3le +++ b/c/Mf-ti3le @@ -26,15 +26,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -melf_i386 -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -melf_i386 -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m32 ./configure) + (cd ../zlib; CFLAGS="${CFLAGS} -m32" ./configure) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m32" ${MAKE} liblz4.a) diff --git a/c/Mf-ti3nb b/c/Mf-ti3nb index fd102dce25..3bbb9d2961 100644 --- a/c/Mf-ti3nb +++ b/c/Mf-ti3nb @@ -27,15 +27,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib ${mdinclude} $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} ${mdinclude} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -m elf_i386 -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -m elf_i386 -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m32 ./configure) + (cd ../zlib; CFLAGS="${CFLAGS} -m32" ./configure) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m32" ${MAKE} liblz4.a) diff --git a/c/Mf-ti3ob b/c/Mf-ti3ob index b6b529fd91..eb622c122a 100644 --- a/c/Mf-ti3ob +++ b/c/Mf-ti3ob @@ -27,15 +27,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib ${mdinclude} $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} ${mdinclude} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -Wl,--export-dynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m32 ./configure) + (cd ../zlib; CFLAGS="${CFLAGS} -m32" ./configure) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m32" ${MAKE} liblz4.a) diff --git a/c/Mf-ti3osx b/c/Mf-ti3osx index f78817db7d..48662d6df3 100644 --- a/c/Mf-ti3osx +++ b/c/Mf-ti3osx @@ -26,15 +26,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m32 ./configure) + (cd ../zlib; CFLAGS="${CFLAGS} -m32" ./configure) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m32" ${MAKE} liblz4.a) diff --git a/c/Mf-ti3s2 b/c/Mf-ti3s2 index 2f9ec15e5f..8430017c6f 100644 --- a/c/Mf-ti3s2 +++ b/c/Mf-ti3s2 @@ -26,15 +26,18 @@ mdobj = i3le.o .SUFFIXES: .c .o .c.o: - $C -c -DSOLARIS -D${Cpu} -I${Include} -I../zlib $*.c + $C -c -DSOLARIS -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -melf_i386 -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -melf_i386 -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m32 ./configure) + (cd ../zlib; CFLAGS="${CFLAGS} -m32" ./configure) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m32" ${MAKE} liblz4.a) diff --git a/c/Mf-tppc32le b/c/Mf-tppc32le index a1a66d97e1..cd01f24ab8 100644 --- a/c/Mf-tppc32le +++ b/c/Mf-tppc32le @@ -26,15 +26,18 @@ mdobj = ppc32le.o .SUFFIXES: .c .o .c.o: - $C -c -D${Cpu} -I${Include} -I../zlib $*.c + $C -c -D${Cpu} -I${Include} ${zlibInc} ${LZ4Inc} $*.c include Mf-base -${Kernel}: ${kernelobj} ../zlib/libz.a - ld -r -X -o ${Kernel} ${kernelobj} ../zlib/libz.a +${Kernel}: ${kernelobj} ${zlibDep} ${LZ4Dep} + ld -r -X -o ${Kernel} ${kernelobj} ${zlibLib} ${LZ4Lib} ${Scheme}: ${Kernel} ${Main} $C -rdynamic -o ${Scheme} ${Kernel} ${Main} ${mdclib} ${LDFLAGS} ../zlib/configure.log: - (cd ../zlib; CFLAGS=-m32 ./configure) + (cd ../zlib; CFLAGS="${CFLAGS} -m32" ./configure) + +../lz4/lib/liblz4.a: ${LZ4Sources} + (cd ../lz4/lib; CFLAGS="${CFLAGS} -m32" ${MAKE} liblz4.a) diff --git a/c/compress.c b/c/compress.c new file mode 100644 index 0000000000..da45c0d699 --- /dev/null +++ b/c/compress.c @@ -0,0 +1,486 @@ +/* system.h + * Copyright 1984-2017 Cisco Systems, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* Dispatch to zlib or LZ4 */ + +#include "system.h" +#include "lz4.h" +#include "lz4frame.h" +#include +#include + +#ifdef WIN32 +# define WIN32_IZE(id) _ ## id +# define GLZ_O_BINARY O_BINARY +#else +# define WIN32_IZE(id) id +# define GLZ_O_BINARY 0 +#endif + +enum { + is_gz, + is_lz4_write, + is_lz4_read +}; + +typedef struct lz4File_out { + int fd; + void *in_buffer, *out_buffer; + int in_pos, out_len, out_pos; + int err; + size_t stream_pos; +} lz4File_out; + +typedef struct lz4File_in { + int fd; + LZ4F_dctx *dctx; + void *in_buffer, *out_buffer; + int in_pos, in_len, out_pos, out_len; + int err; + size_t stream_pos; + off_t init_pos; +} lz4File_in; + +#define USE_LZ4_BUFFER_SIZE (1 << 14) + +static glzFile glzdopen_lz4_pos(int fd, const char *mode, off_t init_pos); + +glzFile glzdopen_gz(int fd, const char *mode) { + gzFile gz; + + if ((gz = gzdopen(fd, mode)) == Z_NULL) { + return Z_NULL; + } else { + glzFile glz = malloc(sizeof(struct glzFile_r)); + glz->mode = is_gz; + glz->gz = gz; + return glz; + } +} + +glzFile glzdopen_lz4(int fd, const char *mode) { + return glzdopen_lz4_pos(fd, mode, 0); +} + +static glzFile glzdopen_lz4_pos(int fd, const char *mode, off_t init_pos) { + glzFile glz = malloc(sizeof(struct glzFile_r)); + + if (mode[0] == 'r') { + LZ4F_dctx *dctx; + LZ4F_errorCode_t r; + lz4File_in *lz4; + + r = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); + if (LZ4F_isError(r)) + return Z_NULL; + + lz4 = malloc(sizeof(lz4File_in)); + lz4->fd = fd; + lz4->dctx = dctx; + lz4->in_buffer = malloc(USE_LZ4_BUFFER_SIZE); + lz4->out_buffer = malloc(USE_LZ4_BUFFER_SIZE); + lz4->in_pos = 0; + lz4->in_len = 0; + lz4->out_len = 0; + lz4->out_pos = 0; + lz4->err = 0; + lz4->stream_pos = 0; + lz4->init_pos = init_pos; + + glz->mode = is_lz4_read; + glz->lz4 = (struct lz4File *)lz4; + } else { + lz4File_out *lz4 = malloc(sizeof(lz4File_out)); + + lz4->fd = fd; + lz4->in_buffer = malloc(USE_LZ4_BUFFER_SIZE); + lz4->out_buffer = malloc(LZ4F_compressFrameBound(USE_LZ4_BUFFER_SIZE, NULL)); + lz4->in_pos = 0; + lz4->out_len = 0; + lz4->out_pos = 0; + lz4->err = 0; + lz4->stream_pos = 0; + + glz->mode = is_lz4_write; + glz->lz4 = (struct lz4File *)lz4; + } + + return glz; +} + +glzFile glzdopen(int fd, const char *mode) { + if (mode[0] == 'r') { + int r, pos = 0; + unsigned char buffer[4]; + off_t init_pos; + + /* check for LZ4 magic number, otherwise defer to gzdopen */ + + init_pos = WIN32_IZE(lseek)(fd, 0, SEEK_CUR); + + while (pos < 4) { + r = WIN32_IZE(read)(fd, (char*)buffer + pos, 4 - pos); + if (r == 0) + break; + else if (r > 0) + pos += r; +#ifdef EINTR + else if (r == EINTR) + r = 0; +#endif + else + break; /* error reading */ + } + + if (pos > 0) + WIN32_IZE(lseek)(fd, init_pos, SEEK_SET); + + if ((pos == 4) + && (buffer[0] == 0x04) + && (buffer[1] == 0x22) + && (buffer[2] == 0x4d) + && (buffer[3] == 0x18)) + return glzdopen_lz4_pos(fd, mode, init_pos); + + return glzdopen_gz(fd, mode); + } else + return glzdopen_gz(fd, mode); +} + +/* currently assumes read mode: */ +glzFile glzopen(const char *path, const char *mode) { + int fd; + + fd = WIN32_IZE(open)(path, O_RDONLY | GLZ_O_BINARY); + + if (fd == -1) + return Z_NULL; + else + return glzdopen(fd, mode); +} + +#ifdef WIN32 +/* currently assumes read mode: */ +glzFile glzopen_w(wchar_t *path, const char *mode) { + int fd; + + fd = _wopen(path, O_RDONLY | GLZ_O_BINARY); + + if (fd == -1) + return Z_NULL; + else + return glzdopen(fd, mode); +} +#endif + +int glzdirect(glzFile file) { + if (file->mode == is_gz) + return gzdirect(file->gz); + else + return 0; +} + +int glzclose(glzFile file) { + if (file->mode == is_gz) { + int r; + r = gzclose(file->gz); + if (r != Z_OK) + return r; + } else if (file->mode == is_lz4_write) { + lz4File_out *lz4 = (lz4File_out *)file->lz4; + glzwrite(file, NULL /* => flush */, 0); + while (1) { + int r = WIN32_IZE(close)(lz4->fd); + if (r == 0) + break; +#ifdef EINTR + else if (r == EINTR) + r = 0; +#endif + else + return r; + } + free(lz4->in_buffer); + free(lz4->out_buffer); + } else { + lz4File_in *lz4 = (lz4File_in *)file->lz4; + while (1) { + int r = WIN32_IZE(close)(lz4->fd); + if (r == 0) + break; +#ifdef EINTR + else if (r == EINTR) + r = 0; +#endif + else + return r; + } + (void)LZ4F_freeDecompressionContext(lz4->dctx); + free(lz4->in_buffer); + free(lz4->out_buffer); + } + free(file); + return Z_OK; +} + +int glzread(glzFile file, void *buffer, unsigned int count) { + if (file->mode == is_gz) { + return gzread(file->gz, buffer, count); + } else { + lz4File_in *lz4 = (lz4File_in *)file->lz4; + + while (lz4->out_pos == lz4->out_len) { + int in_avail; + + in_avail = lz4->in_len - lz4->in_pos; + if (!in_avail) { + while (1) { + in_avail = WIN32_IZE(read)(lz4->fd, (char*)lz4->in_buffer, USE_LZ4_BUFFER_SIZE); + if (in_avail >= 0) { + lz4->in_len = in_avail; + lz4->in_pos = 0; + break; +#ifdef EINTR + } else if (in_avail == EINTR) { + /* try again */ +#endif + } else { + lz4->err = errno; + return -1; + } + } + } + + if (in_avail > 0) { + size_t amt, out_len = USE_LZ4_BUFFER_SIZE, in_len = in_avail; + + /* For a larger enough result buffer, try to decompress directly + to that buffer: */ + if (count >= (out_len >> 1)) { + size_t direct_out_len = count; + amt = LZ4F_decompress(lz4->dctx, + buffer, &direct_out_len, + (char *)lz4->in_buffer + lz4->in_pos, &in_len, + NULL); + + if (LZ4F_isError(amt)) { + lz4->err = amt; + return -1; + } + + lz4->in_pos += in_len; + + if (direct_out_len) { + lz4->stream_pos += direct_out_len; + return direct_out_len; + } + + in_len = in_avail - in_len; + } + + if (in_len > 0) { + amt = LZ4F_decompress(lz4->dctx, + lz4->out_buffer, &out_len, + (char *)lz4->in_buffer + lz4->in_pos, &in_len, + NULL); + + if (LZ4F_isError(amt)) { + lz4->err = amt; + return -1; + } + + lz4->in_pos += in_len; + lz4->out_len = out_len; + lz4->out_pos = 0; + } + } else { + /* EOF on read */ + break; + } + } + + if (lz4->out_pos < lz4->out_len) { + unsigned amt = lz4->out_len - lz4->out_pos; + if (amt > count) amt = count; + memcpy(buffer, (char *)lz4->out_buffer + lz4->out_pos, amt); + lz4->out_pos += amt; + lz4->stream_pos += amt; + return amt; + } + + return 0; + } +} + +int glzwrite(glzFile file, void *buffer, unsigned int count) { + if (file->mode == is_gz) + return gzwrite(file->gz, buffer, count); + else { + lz4File_out *lz4 = (lz4File_out *)file->lz4; + + if ((lz4->in_pos == USE_LZ4_BUFFER_SIZE) + || ((lz4->in_pos > 0) && !buffer)) { + size_t out_len; + + out_len = LZ4F_compressFrame(lz4->out_buffer, LZ4F_compressFrameBound(USE_LZ4_BUFFER_SIZE, NULL), + lz4->in_buffer, lz4->in_pos, + NULL); + if (LZ4F_isError(out_len)) { + lz4->err = out_len; + return -1; + } + + lz4->in_pos = 0; + lz4->out_len = out_len; + lz4->out_pos = 0; + } + + while (lz4->out_pos < lz4->out_len) { + int r = WIN32_IZE(write)(lz4->fd, (char*)lz4->out_buffer + lz4->out_pos, lz4->out_len - lz4->out_pos); + if (r >= 0) + lz4->out_pos += r; +#ifdef EINTR + else if (r == EINTR) + lz4->out_pos += 0; /* try again */ +#endif + else { + lz4->err = errno; + return r; + } + } + + { + unsigned int amt = (USE_LZ4_BUFFER_SIZE - lz4->in_pos); + + if (count < amt) + amt = count; + + memcpy((char *)lz4->in_buffer + lz4->in_pos, buffer, amt); + lz4->in_pos += amt; + lz4->stream_pos += amt; + + return amt; + } + } +} + +long glzseek(glzFile file, long offset, int whence) { + if (file->mode == is_gz) + return gzseek(file->gz, offset, whence); + else if (file->mode == is_lz4_write) { + lz4File_out *lz4 = (lz4File_out *)file->lz4; + if (whence == SEEK_CUR) + offset += lz4->stream_pos; + if (offset >= 0) { + while ((size_t)offset > lz4->stream_pos) { + size_t amt = (size_t)offset - lz4->stream_pos; + if (amt > 8) amt = 8; + if (glzwrite(file, "\0\0\0\0\0\0\0\0", amt) < 0) + return -1; + } + } + return lz4->stream_pos; + } else if (file->mode == is_lz4_read) { + lz4File_in *lz4 = (lz4File_in *)file->lz4; + if (whence == SEEK_CUR) + offset += lz4->stream_pos; + if (offset < 0) + offset = 0; + if ((size_t)offset < lz4->stream_pos) { + /* rewind and read from start */ + if (WIN32_IZE(lseek)(lz4->fd, lz4->init_pos, SEEK_SET) < 0) { + lz4->err = errno; + return -1; + } + LZ4F_resetDecompressionContext(lz4->dctx); + lz4->in_pos = 0; + lz4->in_len = 0; + lz4->out_len = 0; + lz4->out_pos = 0; + lz4->err = 0; + lz4->stream_pos = 0; + } + while ((size_t)offset > lz4->stream_pos) { + char buffer[32]; + size_t amt = (size_t)offset - lz4->stream_pos; + if (amt > sizeof(buffer)) amt = sizeof(buffer); + if (glzread(file, buffer, amt) < 0) + return -1; + } + return lz4->stream_pos; + } else + return 0; +} + +int glzgetc(glzFile file) { + if (file->mode == is_gz) + return gzgetc(file->gz); + else { + unsigned char buffer[1]; + int r; + r = glzread(file, buffer, 1); + if (r == 1) + return buffer[0]; + return -1; + } +} + +int glzungetc(int c, glzFile file) { + if (file->mode == is_gz) + return gzungetc(c, file->gz); + else if (file->mode == is_lz4_read) { + lz4File_in *lz4 = (lz4File_in *)file->lz4; + if (lz4->out_len == 0) + lz4->out_len = lz4->out_pos = 1; + if (lz4->out_pos) { + lz4->out_pos--; + ((unsigned char *)lz4->out_buffer)[lz4->out_pos] = c; + lz4->stream_pos--; + return c; + } else { + /* support ungetc only just after a getc, in which case there + should have been room */ + return -1; + } + } else + return -1; +} + +int glzrewind(glzFile file) { + return glzseek(file, 0, SEEK_SET); +} + +void glzerror(glzFile file, int *errnum) +{ + if (file->mode == is_gz) + (void)gzerror(file->gz, errnum); + else if (file->mode == is_lz4_write) + *errnum = ((lz4File_out *)file->lz4)->err; + else if (file->mode == is_lz4_read) + *errnum = ((lz4File_in *)file->lz4)->err; + else + *errnum = 0; +} + +void glzclearerr(glzFile file) +{ + if (file->mode == is_gz) + gzclearerr(file->gz); + else if (file->mode == is_lz4_write) + ((lz4File_out *)file->lz4)->err = 0; + else if (file->mode == is_lz4_read) + ((lz4File_in *)file->lz4)->err = 0; +} diff --git a/c/compress.h b/c/compress.h new file mode 100644 index 0000000000..1abb72e13f --- /dev/null +++ b/c/compress.h @@ -0,0 +1,31 @@ +#include "zlib.h" + +struct lz4File; + +typedef struct glzFile_r { + int mode; + union { + gzFile gz; + struct lz4File *lz4; + }; +} *glzFile; + +glzFile glzdopen_gz(int fd, const char *mode); +glzFile glzdopen_lz4(int fd, const char *mode); +glzFile glzdopen(int fd, const char *mode); +glzFile glzopen(const char *path, const char *mode); +#ifdef WIN32 +glzFile glzopen_w(wchar_t *path, const char *mode); +#endif +int glzdirect(glzFile file); +int glzclose(glzFile file); + +int glzread(glzFile file, void *buffer, unsigned int count); +int glzwrite(glzFile file, void *buffer, unsigned int count); +long glzseek(glzFile file, long offset, int whence); +int glzgetc(glzFile file); +int glzungetc(int c, glzFile file); +int glzrewind(glzFile file); + +void glzerror(glzFile file, int *errnum); +void glzclearerr(glzFile fdfile); diff --git a/c/expeditor.c b/c/expeditor.c index 78f6be3bfb..7d0fc62cf1 100644 --- a/c/expeditor.c +++ b/c/expeditor.c @@ -527,6 +527,7 @@ static void s_ee_write_char(wchar_t c) { } #else /* WIN32 */ +#include #ifdef SOLARIS #define NCURSES_CONST #define CHTYPE int diff --git a/c/externs.h b/c/externs.h index 91a5216418..e92e7f4ce4 100644 --- a/c/externs.h +++ b/c/externs.h @@ -96,7 +96,7 @@ extern void S_fasl_init PROTO((void)); ptr S_fasl_read PROTO((ptr file, IBOOL gzflag, ptr path)); ptr S_bv_fasl_read PROTO((ptr bv, ptr path)); /* S_boot_read's f argument is really gzFile, but zlib.h is not included everywhere */ -ptr S_boot_read PROTO((gzFile file, const char *path)); +ptr S_boot_read PROTO((glzFile file, const char *path)); char *S_format_scheme_version PROTO((uptr n)); char *S_lookup_machine_type PROTO((uptr n)); extern void S_set_code_obj PROTO((char *who, IFASLCODE typ, ptr p, iptr n, @@ -171,19 +171,19 @@ extern IBOOL S_fixedpathp PROTO((const char *inpath)); /* new-io.c */ extern INT S_gzxfile_fd PROTO((ptr x)); -extern gzFile S_gzxfile_gzfile PROTO((ptr x)); +extern glzFile S_gzxfile_gzfile PROTO((ptr x)); extern ptr S_new_open_input_fd PROTO((const char *filename, IBOOL compressed)); extern ptr S_new_open_output_fd PROTO(( const char *filename, INT mode, IBOOL no_create, IBOOL no_fail, IBOOL no_truncate, - IBOOL append, IBOOL lock, IBOOL replace, IBOOL compressed)); + IBOOL append, IBOOL lock, IBOOL replace, IBOOL compressed, IBOOL as_gz)); extern ptr S_new_open_input_output_fd PROTO(( const char *filename, INT mode, IBOOL no_create, IBOOL no_fail, IBOOL no_truncate, IBOOL append, IBOOL lock, IBOOL replace, IBOOL compressed)); extern ptr S_close_fd PROTO((ptr file, IBOOL gzflag)); extern ptr S_compress_input_fd PROTO((INT fd, I64 fp)); -extern ptr S_compress_output_fd PROTO((INT fd)); +extern ptr S_compress_output_fd PROTO((INT fd, IBOOL as_gz)); extern ptr S_bytevector_read PROTO((ptr file, ptr buffer, iptr start, iptr count, IBOOL gzflag)); extern ptr S_bytevector_read_nb PROTO((ptr file, ptr buffer, iptr start, iptr count, IBOOL gzflag)); @@ -198,11 +198,13 @@ extern ptr S_get_fd_length PROTO((ptr file, IBOOL gzflag)); extern ptr S_set_fd_length PROTO((ptr file, ptr length, IBOOL gzflag)); extern void S_new_io_init PROTO((void)); -extern uptr S_bytevector_compress_size PROTO((iptr s_count)); +extern uptr S_bytevector_compress_size PROTO((iptr s_count, IBOOL as_gz)); extern ptr S_bytevector_compress PROTO((ptr dest_bv, iptr d_start, iptr d_count, - ptr src_bv, iptr s_start, iptr s_count)); + ptr src_bv, iptr s_start, iptr s_count, + IBOOL as_gz)); extern ptr S_bytevector_uncompress PROTO((ptr dest_bv, iptr d_start, iptr d_count, - ptr src_bv, iptr s_start, iptr s_count)); + ptr src_bv, iptr s_start, iptr s_count, + IBOOL as_gz)); /* thread.c */ extern void S_thread_init PROTO((void)); diff --git a/c/fasl.c b/c/fasl.c index ec7799b9bc..1d23ceb97f 100644 --- a/c/fasl.c +++ b/c/fasl.c @@ -195,7 +195,7 @@ typedef struct unbufFaslFileObj { ptr path; INT type; INT fd; - gzFile file; + glzFile file; } *unbufFaslFile; typedef struct faslFileObj { @@ -317,7 +317,7 @@ ptr S_bv_fasl_read(ptr bv, ptr path) { return x; } -ptr S_boot_read(gzFile file, const char *path) { +ptr S_boot_read(glzFile file, const char *path) { ptr tc = get_thread_context(); struct unbufFaslFileObj uffo; @@ -346,14 +346,14 @@ static INT uf_read(unbufFaslFile uf, octet *s, iptr n) { switch (uf->type) { case UFFO_TYPE_GZ: - k = gzread(uf->file, s, (GZ_IO_SIZE_T)nx); + k = glzread(uf->file, s, (GZ_IO_SIZE_T)nx); if (k > 0) n -= k; else if (k == 0) return -1; else { - gzerror(uf->file, &errnum); - gzclearerr(uf->file); + glzerror(uf->file, &errnum); + glzclearerr(uf->file); if (errnum != Z_ERRNO || errno != EINTR) S_error1("", "error reading from ~a", uf->path); } @@ -370,6 +370,8 @@ static INT uf_read(unbufFaslFile uf, octet *s, iptr n) { default: return -1; } + + s += k; } return 0; } diff --git a/c/new-io.c b/c/new-io.c index bcf4779564..f8243bf505 100644 --- a/c/new-io.c +++ b/c/new-io.c @@ -27,6 +27,7 @@ #endif /* WIN32 */ #include #include "zlib.h" +#include "lz4.h" /* !!! UNLESS you enjoy spending endless days tracking down race conditions !!! involving the garbage collector, please note: DEACTIVATE and @@ -51,16 +52,17 @@ /* locally defined functions */ static ptr new_open_output_fd_helper PROTO((const char *filename, INT mode, INT flags, INT no_create, INT no_fail, INT no_truncate, - INT append, INT lock, INT replace, INT compressed)); + INT append, INT lock, INT replace, INT compressed, INT as_gz)); static INT lockfile PROTO((INT fd)); -static ptr make_gzxfile PROTO((int fd, gzFile file)); +static ptr make_gzxfile PROTO((int fd, glzFile file)); static int is_valid_zlib_length(iptr count); +static int is_valid_lz4_length(iptr count); /* - not_ok_is_fatal: !ok definitely implies error, so ignore gzerror + not_ok_is_fatal: !ok definitely implies error, so ignore glzerror ok: whether the result of body seems to be ok flag: will be set when an error is detected and cleared if no error - fd: the gzFile object to call gzerror on + fd: the glzFile object to call glzerror on body: the operation we are checking the error on */ #ifdef EINTR @@ -76,8 +78,8 @@ static int is_valid_zlib_length(iptr count); if (ok) { flag = 0; } \ else { \ INT errnum; \ - gzerror((fd),&errnum); \ - gzclearerr((fd)); \ + glzerror((fd),&errnum); \ + glzclearerr((fd)); \ if (errnum == Z_ERRNO) { \ flag = errno != EINTR; \ } else { \ @@ -97,8 +99,8 @@ static int is_valid_zlib_length(iptr count); if (ok) { flag = 0; break; } \ else { \ INT errnum; \ - gzerror((fd),&errnum); \ - gzclearerr((fd)); \ + glzerror((fd),&errnum); \ + glzclearerr((fd)); \ if (errnum == Z_ERRNO) { \ if (errno != EINTR) { flag = 1; break; } \ } else { \ @@ -115,8 +117,8 @@ static int is_valid_zlib_length(iptr count); if (ok) { flag = 0; } \ else { \ INT errnum; \ - gzerror((fd),&errnum); \ - gzclearerr((fd)); \ + glzerror((fd),&errnum); \ + glzclearerr((fd)); \ if (errnum == Z_ERRNO) { flag = 1; } \ else { \ flag = not_ok_is_fatal || errnum != Z_OK; \ @@ -143,14 +145,14 @@ static INT lockfile(INT fd) { return FLOCK(fd, LOCK_EX); } static INT lockfile(INT fd) { return lockf(fd, F_LOCK, (off_t)0); } #endif -/* work around missing zlib API operation to extract a gzFile's fd */ +/* work around missing zlib API operation to extract a glzFile's fd */ typedef struct { int fd; - gzFile file; + glzFile file; } gzxfile; #define gzxfile_fd(x) (((gzxfile *)&BVIT(x,0))->fd) #define gzxfile_gzfile(x) (((gzxfile *)&BVIT(x,0))->file) -static ptr make_gzxfile(int fd, gzFile file) { +static ptr make_gzxfile(int fd, glzFile file) { ptr bv; bv = S_bytevector(sizeof(gzxfile)); @@ -161,7 +163,7 @@ static ptr make_gzxfile(int fd, gzFile file) { INT S_gzxfile_fd(ptr x) { return gzxfile_fd(x); } -gzFile S_gzxfile_gzfile(ptr x) { +glzFile S_gzxfile_gzfile(ptr x) { return gzxfile_gzfile(x); } @@ -169,7 +171,7 @@ ptr S_new_open_input_fd(const char *infilename, IBOOL compressed) { char *filename; INT saved_errno = 0; INT fd, dupfd, error, result, ok, flag; - gzFile file; + glzFile file; #ifdef PTHREADS ptr tc = get_thread_context(); #endif @@ -207,14 +209,14 @@ ptr S_new_open_input_fd(const char *infilename, IBOOL compressed) { return Scons(FIX(OPEN_ERROR_OTHER), str); } - if ((file = gzdopen(dupfd, "rb")) == Z_NULL) { + if ((file = glzdopen(dupfd, "rb")) == Z_NULL) { FD_GUARD(result == 0, error, result = CLOSE(fd)); FD_GUARD(result == 0, error, result = CLOSE(dupfd)); return Scons(FIX(OPEN_ERROR_OTHER), Sstring("unable to allocate compression state (too many open files?)")); } DEACTIVATE(tc) - compressed = !gzdirect(file); + compressed = !glzdirect(file); REACTIVATE(tc) if (compressed) { @@ -223,9 +225,9 @@ ptr S_new_open_input_fd(const char *infilename, IBOOL compressed) { return Sbox(make_gzxfile(dupfd, file)); } - GZ_GUARD(1, ok == 0 || ok == Z_BUF_ERROR, flag, file, ok = gzclose(file)); + GZ_GUARD(1, ok == 0 || ok == Z_BUF_ERROR, flag, file, ok = glzclose(file)); if (flag) {} /* make the compiler happy */ - if (LSEEK(fd, 0, SEEK_SET) != 0) { /* gzdirect does not leave fd at position 0 */ + if (LSEEK(fd, 0, SEEK_SET) != 0) { /* glzdirect does not leave fd at position 0 */ FD_GUARD(result == 0, error, result = CLOSE(fd)); return Scons(FIX(OPEN_ERROR_OTHER),Sstring("unable to reset after reading header bytes")); } @@ -234,7 +236,7 @@ ptr S_new_open_input_fd(const char *infilename, IBOOL compressed) { ptr S_compress_input_fd(INT fd, I64 pos) { INT dupfd, error, result, ok, flag; IBOOL compressed; - gzFile file; + glzFile file; #ifdef PTHREADS ptr tc = get_thread_context(); #endif @@ -243,13 +245,13 @@ ptr S_compress_input_fd(INT fd, I64 pos) { return S_strerror(errno); } - if ((file = gzdopen(dupfd, "rb")) == Z_NULL) { + if ((file = glzdopen(dupfd, "rb")) == Z_NULL) { FD_GUARD(result == 0, error, result = CLOSE(dupfd)); return Sstring("unable to allocate compression state (too many open files?)"); } DEACTIVATE(tc) - compressed = !gzdirect(file); + compressed = !glzdirect(file); REACTIVATE(tc) if (compressed) { @@ -258,24 +260,31 @@ ptr S_compress_input_fd(INT fd, I64 pos) { return Sbox(make_gzxfile(dupfd, file)); } - GZ_GUARD(1, ok == 0 || ok == Z_BUF_ERROR, flag, file, ok = gzclose(file)); + GZ_GUARD(1, ok == 0 || ok == Z_BUF_ERROR, flag, file, ok = glzclose(file)); if (flag) {} /* make the compiler happy */ - if (LSEEK(fd, pos, SEEK_SET) != pos) { /* gzdirect does not leave fd at same position */ + if (LSEEK(fd, pos, SEEK_SET) != pos) { /* glzdirect does not leave fd at same position */ return Sstring("unable to reset after reading header bytes"); } return MAKE_FD(fd); } -ptr S_compress_output_fd(INT fd) { - gzFile file; +ptr S_compress_output_fd(INT fd, IBOOL as_gz) { + glzFile file; + int as_append; #ifdef WIN32 - if ((file = gzdopen(fd, "wb")) == Z_NULL) { + as_append = 0; #else - if ((file = gzdopen(fd, (fcntl(fd, F_GETFL) & O_APPEND) ? "ab" : "wb")) == Z_NULL) { + as_append = fcntl(fd, F_GETFL) & O_APPEND; #endif + + if (as_gz) + file = glzdopen_gz(fd, as_append ? "ab" : "wb"); + else + file = glzdopen_lz4(fd, as_append ? "ab" : "wb"); + + if (file == Z_NULL) return Sstring("unable to allocate compression state (too many open files?)"); - } return Sbox(make_gzxfile(fd, file)); } @@ -283,7 +292,7 @@ ptr S_compress_output_fd(INT fd) { static ptr new_open_output_fd_helper( const char *infilename, INT mode, INT flags, IBOOL no_create, IBOOL no_fail, IBOOL no_truncate, - IBOOL append, IBOOL lock, IBOOL replace, IBOOL compressed) { + IBOOL append, IBOOL lock, IBOOL replace, IBOOL compressed, IBOOL as_gz) { char *filename; INT saved_errno = 0; iptr error; @@ -347,8 +356,12 @@ static ptr new_open_output_fd_helper( if (!compressed) { return MAKE_FD(fd); } - - gzFile file = gzdopen(fd, append ? "ab" : "wb"); + + glzFile file; + if (as_gz) + file = glzdopen_gz(fd, append ? "ab" : "wb"); + else + file = glzdopen_lz4(fd, append ? "ab" : "wb"); if (file == Z_NULL) { FD_GUARD(result == 0, error, result = CLOSE(fd)); return Scons(FIX(OPEN_ERROR_OTHER), Sstring("unable to allocate compression state")); @@ -360,11 +373,11 @@ static ptr new_open_output_fd_helper( ptr S_new_open_output_fd( const char *filename, INT mode, IBOOL no_create, IBOOL no_fail, IBOOL no_truncate, - IBOOL append, IBOOL lock, IBOOL replace, IBOOL compressed) { + IBOOL append, IBOOL lock, IBOOL replace, IBOOL compressed, IBOOL as_gz) { return new_open_output_fd_helper( filename, mode, O_BINARY | O_WRONLY, no_create, no_fail, no_truncate, - append, lock, replace, compressed); + append, lock, replace, compressed, as_gz); } ptr S_new_open_input_output_fd( @@ -377,14 +390,14 @@ ptr S_new_open_input_output_fd( return new_open_output_fd_helper( filename, mode, O_BINARY | O_RDWR, no_create, no_fail, no_truncate, - append, lock, replace, compressed); + append, lock, replace, compressed, 0); } ptr S_close_fd(ptr file, IBOOL gzflag) { INT saved_errno = 0; INT ok, flag; INT fd = gzflag ? 0 : GET_FD(file); - gzFile gzfile = gzflag ? gzxfile_gzfile(file) : NULL; + glzFile gzfile = gzflag ? gzxfile_gzfile(file) : NULL; #ifdef PTHREADS ptr tc = get_thread_context(); #endif @@ -401,7 +414,7 @@ ptr S_close_fd(ptr file, IBOOL gzflag) { FD_GUARD(ok == 0, flag, ok = CLOSE(fd)); } else { /* zlib 1.2.1 returns Z_BUF_ERROR when closing an empty file opened for reading */ - GZ_GUARD(1, ok == 0 || ok == Z_BUF_ERROR, flag, gzfile, ok = gzclose(gzfile)); + GZ_GUARD(1, ok == 0 || ok == Z_BUF_ERROR, flag, gzfile, ok = glzclose(gzfile)); } saved_errno = errno; REACTIVATE(tc) @@ -431,7 +444,7 @@ ptr S_bytevector_read(ptr file, ptr bv, iptr start, iptr count, IBOOL gzflag) { ptr tc = get_thread_context(); iptr m, flag = 0; INT fd = gzflag ? 0 : GET_FD(file); - gzFile gzfile = gzflag ? gzxfile_gzfile(file) : NULL; + glzFile gzfile = gzflag ? gzxfile_gzfile(file) : NULL; /* file is not locked; do not reference after deactivating thread! */ file = (ptr)-1; @@ -464,7 +477,7 @@ ptr S_bytevector_read(ptr file, ptr bv, iptr start, iptr count, IBOOL gzflag) { GZ_EINTR_GUARD( 1, m >= 0 || Sboolean_value(KEYBOARDINTERRUPTPENDING(tc)), flag, gzfile, - m = gzread(gzfile, &BVIT(bv,start), (GZ_IO_SIZE_T)count)); + m = glzread(gzfile, &BVIT(bv,start), (GZ_IO_SIZE_T)count)); } } saved_errno = errno; @@ -548,7 +561,7 @@ ptr S_bytevector_write(ptr file, ptr bv, iptr start, iptr count, IBOOL gzflag) { ptr tc = get_thread_context(); INT flag = 0, saved_errno = 0; INT fd = gzflag ? 0 : GET_FD(file); - gzFile gzfile = gzflag ? gzxfile_gzfile(file) : NULL; + glzFile gzfile = gzflag ? gzxfile_gzfile(file) : NULL; for (s = start, c = count; c > 0; s += i, c -= i) { iptr cx = c; @@ -566,7 +579,7 @@ ptr S_bytevector_write(ptr file, ptr bv, iptr start, iptr count, IBOOL gzflag) { GZ_EINTR_GUARD( i < 0, i > 0 || Sboolean_value(KEYBOARDINTERRUPTPENDING(tc)), flag, gzfile, - i = gzwrite(gzfile, &BVIT(bv,s), (GZ_IO_SIZE_T)cx)); + i = glzwrite(gzfile, &BVIT(bv,s), (GZ_IO_SIZE_T)cx)); } else { FD_EINTR_GUARD(i >= 0 || Sboolean_value(KEYBOARDINTERRUPTPENDING(tc)), flag, i = WRITE(fd, &BVIT(bv,s), (IO_SIZE_T)cx)); @@ -610,7 +623,7 @@ ptr S_put_byte(ptr file, INT byte, IBOOL gzflag) { ptr tc = get_thread_context(); INT flag = 0, saved_errno = 0; INT fd = gzflag ? 0 : GET_FD(file); - gzFile gzfile = gzflag ? gzxfile_gzfile(file) : NULL; + glzFile gzfile = gzflag ? gzxfile_gzfile(file) : NULL; octet buf[1]; buf[0] = (octet)byte; @@ -621,7 +634,7 @@ ptr S_put_byte(ptr file, INT byte, IBOOL gzflag) { GZ_EINTR_GUARD( i < 0, i > 0 || Sboolean_value(KEYBOARDINTERRUPTPENDING(tc)), flag, gzfile, - i = gzwrite(gzfile, buf, 1)); + i = glzwrite(gzfile, buf, 1)); } else { FD_EINTR_GUARD(i >= 0 || Sboolean_value(KEYBOARDINTERRUPTPENDING(tc)), flag, i = WRITE(fd, buf, 1)); @@ -651,7 +664,7 @@ ptr S_put_byte(ptr file, INT byte, IBOOL gzflag) { ptr S_get_fd_pos(ptr file, IBOOL gzflag) { errno = 0; if (gzflag) { - z_off_t offset = gzseek(gzxfile_gzfile(file), 0, SEEK_CUR); + z_off_t offset = glzseek(gzxfile_gzfile(file), 0, SEEK_CUR); if (offset != -1) return Sinteger64(offset); } else { OFF_T offset = LSEEK(GET_FD(file), 0, SEEK_CUR); @@ -670,7 +683,7 @@ ptr S_set_fd_pos(ptr file, ptr pos, IBOOL gzflag) { if (sizeof(z_off_t) != sizeof(I64)) if (offset != offset64) return Sstring("invalid position"); errno = 0; - if (gzseek(gzxfile_gzfile(file),offset,SEEK_SET) == offset) return Strue; + if (glzseek(gzxfile_gzfile(file),offset,SEEK_SET) == offset) return Strue; if (errno == 0) return Sstring("compression failed"); return S_strerror(errno); } else { @@ -791,58 +804,103 @@ static int is_valid_zlib_length(iptr count) { return count == (iptr)(uLong)count; } +static int is_valid_lz4_length(iptr len) { + return (len <= LZ4_MAX_INPUT_SIZE); +} + /* Accept `iptr` because we expect it to represent a bytevector size, which always fits in `iptr`. Return `uptr`, because the result might not fit in `iptr`. */ -uptr S_bytevector_compress_size(iptr s_count) { - if (is_valid_zlib_length(s_count)) - return compressBound((uLong)s_count); - else { - /* Compression will report "source too long" */ - return 0; +uptr S_bytevector_compress_size(iptr s_count, IBOOL as_gz) { + if (as_gz) { + if (is_valid_zlib_length(s_count)) + return compressBound((uLong)s_count); + else { + /* Compression will report "source too long" */ + return 0; + } + } else { + if (is_valid_lz4_length(s_count)) + return LZ4_compressBound((uLong)s_count); + else { + /* Compression will report "source too long" */ + return 0; + } } } ptr S_bytevector_compress(ptr dest_bv, iptr d_start, iptr d_count, - ptr src_bv, iptr s_start, iptr s_count) { + ptr src_bv, iptr s_start, iptr s_count, + IBOOL as_gz) { /* On error, an message-template string with ~s for the bytevector */ - int r; - uLong destLen; + if (as_gz) { + int r; + uLong destLen; + + if (!is_valid_zlib_length(s_count)) + return Sstring("source bytevector ~s is too large"); + + destLen = (uLong)d_count; + + r = compress(&BVIT(dest_bv, d_start), &destLen, &BVIT(src_bv, s_start), (uLong)s_count); + + if (r == Z_OK) + return FIX(destLen); + else if (r == Z_BUF_ERROR) + return Sstring("destination bytevector is too small for compressed form of ~s"); + else + return Sstring("internal error compressing ~s"); + } else { + int destLen; - if (!is_valid_zlib_length(s_count)) - return Sstring("source bytevector ~s is too large"); + if (!is_valid_lz4_length(s_count)) + return Sstring("source bytevector ~s is too large"); - destLen = (uLong)d_count; + destLen = (int)d_count; - r = compress(&BVIT(dest_bv, d_start), &destLen, &BVIT(src_bv, s_start), (uLong)s_count); + destLen = LZ4_compress_default((char *)&BVIT(src_bv, s_start), (char *)&BVIT(dest_bv, d_start), (int)s_count, (int)d_count); - if (r == Z_OK) - return FIX(destLen); - else if (r == Z_BUF_ERROR) - return Sstring("destination bytevector is too small for compressed form of ~s"); - else - return Sstring("internal error compressing ~s"); + if (destLen > 0) + return Sfixnum(destLen); + else + return Sstring("compression failed for ~s"); + } } ptr S_bytevector_uncompress(ptr dest_bv, iptr d_start, iptr d_count, - ptr src_bv, iptr s_start, iptr s_count) { + ptr src_bv, iptr s_start, iptr s_count, + IBOOL as_gz) { /* On error, an message-template string with ~s for the bytevector */ - int r; - uLongf destLen; + if (as_gz) { + int r; + uLongf destLen; - if (!is_valid_zlib_length(d_count)) - return Sstring("expected result size of uncompressed source ~s is too large"); + if (!is_valid_zlib_length(d_count)) + return Sstring("expected result size of uncompressed source ~s is too large"); - destLen = (uLongf)d_count; + destLen = (uLongf)d_count; - r = uncompress(&BVIT(dest_bv, d_start), &destLen, &BVIT(src_bv, s_start), (uLong)s_count); + r = uncompress(&BVIT(dest_bv, d_start), &destLen, &BVIT(src_bv, s_start), (uLong)s_count); - if (r == Z_OK) - return FIX(destLen); - else if (r == Z_BUF_ERROR) - return Sstring("uncompressed ~s is larger than expected size"); - else if (r == Z_DATA_ERROR) - return Sstring("invalid data in source bytevector ~s"); - else - return Sstring("internal error uncompressing ~s"); + if (r == Z_OK) + return FIX(destLen); + else if (r == Z_BUF_ERROR) + return Sstring("uncompressed ~s is larger than expected size"); + else if (r == Z_DATA_ERROR) + return Sstring("invalid data in source bytevector ~s"); + else + return Sstring("internal error uncompressing ~s"); + } else { + int r; + + if (!is_valid_lz4_length(d_count)) + return Sstring("expected result size of uncompressed source ~s is too large"); + + r = LZ4_decompress_safe((char *)&BVIT(src_bv, s_start), (char *)&BVIT(dest_bv, d_start), (int)s_count, (int)d_count); + + if (r >= 0) + return Sfixnum(r); + else + return Sstring("internal error uncompressing ~s"); + } } diff --git a/c/scheme.c b/c/scheme.c index 4f14cbcd66..1dd9c86b66 100644 --- a/c/scheme.c +++ b/c/scheme.c @@ -551,7 +551,7 @@ static IBOOL next_path(path, name, ext, sp, dsp) char *path; const char *name, * /* BOOT FILES */ typedef struct { - gzFile file; + glzFile file; char path[PATH_MAX]; } boot_desc; @@ -559,8 +559,8 @@ typedef struct { static boot_desc bd[MAX_BOOT_FILES]; /* locally defined functions */ -static uptr zget_uptr PROTO((gzFile file, uptr *pn)); -static INT zgetstr PROTO((gzFile file, char *s, iptr max)); +static uptr zget_uptr PROTO((glzFile file, uptr *pn)); +static INT zgetstr PROTO((glzFile file, char *s, iptr max)); static IBOOL find_boot PROTO((const char *name, const char *ext, int fd, IBOOL errorp)); static void load PROTO((ptr tc, iptr n, IBOOL base)); static void check_boot_file_state PROTO((const char *who)); @@ -574,7 +574,7 @@ static IBOOL find_boot(name, ext, fd, errorp) const char *name, *ext; int fd; IB #else char *expandedpath; #endif - gzFile file; + glzFile file; if ((fd != -1) || S_fixedpathp(name)) { if (strlen(name) >= PATH_MAX) { @@ -585,17 +585,17 @@ static IBOOL find_boot(name, ext, fd, errorp) const char *name, *ext; int fd; IB path = name; if (fd != -1) { - file = gzdopen(fd, "rb"); + file = glzdopen(fd, "rb"); } else { #ifdef WIN32 expandedpath = S_malloc_wide_pathname(path); - file = gzopen_w(expandedpath, "rb"); + file = glzopen_w(expandedpath, "rb"); #else expandedpath = S_malloc_pathname(path); - file = gzopen(expandedpath, "rb"); + file = glzopen(expandedpath, "rb"); #endif /* assumption (seemingly true based on a glance at the source code): - gzopen doesn't squirrel away a pointer to expandedpath. */ + glzopen doesn't squirrel away a pointer to expandedpath. */ free(expandedpath); } @@ -611,14 +611,14 @@ static IBOOL find_boot(name, ext, fd, errorp) const char *name, *ext; int fd; IB if (verbose) fprintf(stderr, "trying %s...opened\n", path); /* check for magic number */ - if (gzgetc(file) != fasl_type_header || - gzgetc(file) != 0 || - gzgetc(file) != 0 || - gzgetc(file) != 0 || - gzgetc(file) != 'c' || - gzgetc(file) != 'h' || - gzgetc(file) != 'e' || - gzgetc(file) != 'z') { + if (glzgetc(file) != fasl_type_header || + glzgetc(file) != 0 || + glzgetc(file) != 0 || + glzgetc(file) != 0 || + glzgetc(file) != 'c' || + glzgetc(file) != 'h' || + glzgetc(file) != 'e' || + glzgetc(file) != 'z') { fprintf(stderr, "malformed fasl-object header in %s\n", path); S_abnormal_exit(); } @@ -626,7 +626,7 @@ static IBOOL find_boot(name, ext, fd, errorp) const char *name, *ext; int fd; IB /* check version */ if (zget_uptr(file, &n) != 0) { fprintf(stderr, "unexpected end of file on %s\n", path); - gzclose(file); + glzclose(file); S_abnormal_exit(); } @@ -634,21 +634,21 @@ static IBOOL find_boot(name, ext, fd, errorp) const char *name, *ext; int fd; IB fprintf(stderr, "%s is for Version %s; ", path, S_format_scheme_version(n)); /* use separate fprintf since S_format_scheme_version returns static string */ fprintf(stderr, "need Version %s\n", S_format_scheme_version(scheme_version)); - gzclose(file); + glzclose(file); S_abnormal_exit(); } /* check machine type */ if (zget_uptr(file, &n) != 0) { fprintf(stderr, "unexpected end of file on %s\n", path); - gzclose(file); + glzclose(file); S_abnormal_exit(); } if (n != machine_type) { fprintf(stderr, "%s is for machine-type %s; need machine-type %s\n", path, S_lookup_machine_type(n), S_lookup_machine_type(machine_type)); - gzclose(file); + glzclose(file); S_abnormal_exit(); } } else { @@ -671,13 +671,13 @@ static IBOOL find_boot(name, ext, fd, errorp) const char *name, *ext; int fd; IB #ifdef WIN32 expandedpath = S_malloc_wide_pathname(path); - file = gzopen_w(expandedpath, "rb"); + file = glzopen_w(expandedpath, "rb"); #else expandedpath = S_malloc_pathname(path); - file = gzopen(expandedpath, "rb"); + file = glzopen(expandedpath, "rb"); #endif /* assumption (seemingly true based on a glance at the source code): - gzopen doesn't squirrel away a pointer to expandedpath. */ + glzopen doesn't squirrel away a pointer to expandedpath. */ free(expandedpath); if (!file) { if (verbose) fprintf(stderr, "trying %s...cannot open\n", path); @@ -687,23 +687,23 @@ static IBOOL find_boot(name, ext, fd, errorp) const char *name, *ext; int fd; IB if (verbose) fprintf(stderr, "trying %s...opened\n", path); /* check for magic number */ - if (gzgetc(file) != fasl_type_header || - gzgetc(file) != 0 || - gzgetc(file) != 0 || - gzgetc(file) != 0 || - gzgetc(file) != 'c' || - gzgetc(file) != 'h' || - gzgetc(file) != 'e' || - gzgetc(file) != 'z') { + if (glzgetc(file) != fasl_type_header || + glzgetc(file) != 0 || + glzgetc(file) != 0 || + glzgetc(file) != 0 || + glzgetc(file) != 'c' || + glzgetc(file) != 'h' || + glzgetc(file) != 'e' || + glzgetc(file) != 'z') { if (verbose) fprintf(stderr, "malformed fasl-object header in %s\n", path); - gzclose(file); + glzclose(file); continue; } /* check version */ if (zget_uptr(file, &n) != 0) { if (verbose) fprintf(stderr, "unexpected end of file on %s\n", path); - gzclose(file); + glzclose(file); continue; } @@ -713,14 +713,14 @@ static IBOOL find_boot(name, ext, fd, errorp) const char *name, *ext; int fd; IB /* use separate fprintf since S_format_scheme_version returns static string */ fprintf(stderr, "need Version %s\n", S_format_scheme_version(scheme_version)); } - gzclose(file); + glzclose(file); continue; } /* check machine type */ if (zget_uptr(file, &n) != 0) { if (verbose) fprintf(stderr, "unexpected end of file on %s\n", path); - gzclose(file); + glzclose(file); continue; } @@ -728,7 +728,7 @@ static IBOOL find_boot(name, ext, fd, errorp) const char *name, *ext; int fd; IB if (verbose) fprintf(stderr, "%s is for machine-type %s; need machine-type %s\n", path, S_lookup_machine_type(n), S_lookup_machine_type(machine_type)); - gzclose(file); + glzclose(file); continue; } @@ -738,56 +738,56 @@ static IBOOL find_boot(name, ext, fd, errorp) const char *name, *ext; int fd; IB if (verbose) fprintf(stderr, "version and machine type check\n"); - if (gzgetc(file) != '(') { /* ) */ + if (glzgetc(file) != '(') { /* ) */ fprintf(stderr, "malformed boot file %s\n", path); - gzclose(file); + glzclose(file); S_abnormal_exit(); } /* ( */ - if ((c = gzgetc(file)) == ')') { + if ((c = glzgetc(file)) == ')') { if (boot_count != 0) { fprintf(stderr, "base boot file %s must come before other boot files\n", path); - gzclose(file); + glzclose(file); S_abnormal_exit(); } } else { if (boot_count == 0) { for (;;) { - gzungetc(c, file); + glzungetc(c, file); /* try to load heap or boot file this boot file requires */ if (zgetstr(file, buf, PATH_MAX) != 0) { fprintf(stderr, "unexpected end of file on %s\n", path); - gzclose(file); + glzclose(file); S_abnormal_exit(); } if (find_boot(buf, ".boot", -1, 0)) break; - if ((c = gzgetc(file)) == ')') { + if ((c = glzgetc(file)) == ')') { char *sep; char *wastebuf[8]; fprintf(stderr, "cannot find subordinate boot file "); - gzrewind(file); - (void) gzread(file, wastebuf, 8); /* magic number */ + glzrewind(file); + (void) glzread(file, wastebuf, 8); /* magic number */ (void) zget_uptr(file, &n); /* version */ (void) zget_uptr(file, &n); /* machine type */ - (void) gzgetc(file); /* open paren */ + (void) glzgetc(file); /* open paren */ for (sep = ""; ; sep = "or ") { - if ((c = gzgetc(file)) == ')') break; - gzungetc(c, file); + if ((c = glzgetc(file)) == ')') break; + glzungetc(c, file); (void) zgetstr(file, buf, PATH_MAX); fprintf(stderr, "%s%s.boot ", sep, buf); } fprintf(stderr, "required by %s\n", path); - gzclose(file); + glzclose(file); S_abnormal_exit(); } } } /* skip to end of header */ - while ((c = gzgetc(file)) != ')') { + while ((c = glzgetc(file)) != ')') { if (c < 0) { fprintf(stderr, "malformed boot file %s\n", path); - gzclose(file); + glzclose(file); S_abnormal_exit(); } } @@ -805,14 +805,14 @@ static IBOOL find_boot(name, ext, fd, errorp) const char *name, *ext; int fd; IB return 1; } -static uptr zget_uptr(gzFile file, uptr *pn) { +static uptr zget_uptr(glzFile file, uptr *pn) { uptr n, m; int c; octet k; - if ((c = gzgetc(file)) < 0) return -1; + if ((c = glzgetc(file)) < 0) return -1; k = (octet)c; n = k >> 1; while (k & 1) { - if ((c = gzgetc(file)) < 0) return -1; + if ((c = glzgetc(file)) < 0) return -1; k = (octet)c; m = n << 7; if (m >> 7 != n) return -1; @@ -822,13 +822,13 @@ static uptr zget_uptr(gzFile file, uptr *pn) { return 0; } -static INT zgetstr(file, s, max) gzFile file; char *s; iptr max; { +static INT zgetstr(file, s, max) glzFile file; char *s; iptr max; { ICHAR c; while (max-- > 0) { - if ((c = gzgetc(file)) < 0) return -1; + if ((c = glzgetc(file)) < 0) return -1; if (c == ' ' || c == ')') { - if (c == ')') gzungetc(c, file); + if (c == ')') glzungetc(c, file); *s = 0; return 0; } @@ -923,7 +923,7 @@ static void load(tc, n, base) ptr tc; iptr n; IBOOL base; { } S_G.load_binary = Sfalse; - gzclose(bd[n].file); + glzclose(bd[n].file); } /***************************************************************************/ diff --git a/c/system.h b/c/system.h index 565155aacf..54f9966b2f 100644 --- a/c/system.h +++ b/c/system.h @@ -24,7 +24,7 @@ #include "version.h" #include -#include "zlib.h" +#include "compress.h" #include #include "thread.h" diff --git a/configure b/configure index a62366416b..7c3eaea06b 100755 --- a/configure +++ b/configure @@ -42,6 +42,14 @@ disablex11=no : ${CPPFLAGS:=""} : ${CFLAGS:=""} : ${LDFLAGS:=""} +zlibInc=-I../zlib +LZ4Inc=-I../lz4/lib +zlibDep=../zlib/libz.a +LZ4Dep=../lz4/lib/liblz4.a +zlibLib=../zlib/libz.a +LZ4Lib=../lz4/lib/liblz4.a +zlibHeaderDep="../zlib/zconf.h ../zlib/zlib.h" +LZ4HeaderDep="../lz4/lib/lz4.h ../lz4/lib/lz4frame.h" # On WSL, set OS to "Windows_NT" to create a Windows # build instead of a Linux (on Windows) build: @@ -216,6 +224,18 @@ while [ $# != 0 ] ; do LDFLAGS=*) LDFLAGS=`echo $1 | sed -e 's/^LDFLAGS=//'` ;; + ZLIB=*) + zlibLib=`echo $1 | sed -e 's/^ZLIB=//'` + zlibInc= + zlibDep= + zlibHeaderDep= + ;; + LZ4=*) + LZ4Lib=`echo $1 | sed -e 's/^LZ4=//'` + LZ4Inc= + LZ4Dep= + LZ4HeaderDep= + ;; *) echo "option '$1' unrecognized or missing an argument; try $0 --help" exit 1 @@ -294,6 +314,8 @@ if [ "$help" = "yes" ]; then echo " CPPFLAGS= additional C preprocessor flags ($CPPFLAGS)" echo " CFLAGS= additional C compiler flags ($CFLAGS)" echo " LDFLAGS= additional linker flags ($LDFLAGS)" + echo " ZLIB= link to static instead of own zlib" + echo " LZ4= link to static instead of own LZ4" echo "" echo "Available machine types: $machs" echo "" @@ -331,9 +353,18 @@ else (curl -L -o v1.9.tar.gz https://github.com/nanopass/nanopass-framework-scheme/archive/v1.9.tar.gz && tar -zxf v1.9.tar.gz && mv nanopass-framework-scheme-1.9 nanopass && rm v1.9.tar.gz) || exit 1 fi - if [ ! -f 'zlib/configure' ] ; then - rmdir zlib > /dev/null 2>&1 - (curl -L -o v1.2.11.tar.gz https://github.com/madler/zlib/archive/v1.2.11.tar.gz && tar -xzf v1.2.11.tar.gz && mv zlib-1.2.11 zlib && rm v1.2.11.tar.gz) || exit 1 + if [ "${zlibDep}" != "" ] ; then + if [ ! -f 'zlib/configure' ] ; then + rmdir zlib > /dev/null 2>&1 + (curl -L -o v1.2.11.tar.gz https://github.com/madler/zlib/archive/v1.2.11.tar.gz && tar -xzf v1.2.11.tar.gz && mv zlib-1.2.11 zlib && rm v1.2.11.tar.gz) || exit 1 + fi + fi + + if [ "${LZ4Dep}" != "" ] ; then + if [ ! -f 'lz4/lib/Makefile' ] ; then + rmdir lz4 > /dev/null 2>&1 + (curl -L -o v1.8.3.tar.gz https://github.com/lz4/lz4/archive/v1.8.3.tar.gz && tar -xzf v1.8.3.tar.gz && mv lz4-1.8.3 lz4 && rm v1.8.3.tar.gz) || exit 1 + fi fi if [ ! -f 'stex/Mf-stex' ] ; then @@ -389,4 +420,12 @@ CC=$CC CPPFLAGS=$CPPFLAGS CFLAGS=$CFLAGS LDFLAGS=$LDFLAGS +zlibInc=$zlibInc +LZ4Inc=$LZ4Inc +zlibDep=$zlibDep +LZ4Dep=$LZ4Dep +zlibLib=$zlibLib +LZ4Lib=$LZ4Lib +zlibHeaderDep=$zlibHeaderDep +LZ4HeaderDep=$LZ4HeaderDep END diff --git a/csug/io.stex b/csug/io.stex index 0c0bfc1ea4..a8bf76e4fd 100644 --- a/csug/io.stex +++ b/csug/io.stex @@ -207,7 +207,9 @@ Section~\ref{TSPL:SECTOPENINGFILES} of {\TSPLFOUR}. \begin{description} \item[\var{compressed}:] An output file should be compressed when written; and a compressed input -file should be decompressed when read. +file should be decompressed when read. The compression format for output +is determined by the \scheme{compress-format} parameter, while the compression +format on input is inferred. \item[\var{replace}:] For output files only, replace (remove and recreate) the existing file if @@ -972,8 +974,35 @@ If the port is an output port, subsequent output sent to the port will be compressed. If the port is an input port, subsequent input will be decompressed if and only if the port is currently pointing at compressed data. +The compression format for output +is determined by the \scheme{compress-format} parameter, while the compression +format on input is inferred. This procedure has no effect if the port is already set for compression. +%---------------------------------------------------------------------------- +\entryheader +\formdef{compress-format}{\categorythreadparameter}{compress-format} +\listlibraries +\endnoskipentryheader + +\noindent +\scheme{compress-format} is a parameter that determines the +compression algorithm and format that is used for output. Currently, +the possible values of the parameter are \scheme{'lz4} (the default) +and \scheme{'gzip}. + +The \scheme{'lz4} format uses the LZ4 compression library developed by +Yann Collet. +It is therefore compatible with the \scheme{lz4} program, which +means that \scheme{lz4} may be used to uncompress files produced +by {\ChezScheme} and visa versa. + +The \scheme{'gzip} format uses the zlib compression library developed by +Jean-loup Gailly and Mark Adler. +It is therefore compatible with the \scheme{gzip} program, which +means that \scheme{gzip} may be used to uncompress files produced +by {\ChezScheme} and visa versa. + \section{String Ports\label{SECTIOSTRINGPORTS}} @@ -1224,7 +1253,8 @@ An option list is a list containing zero or more symbolic option names. The mutually exclusive \scheme{compressed} and \scheme{uncompressed} options determine whether the input file -should be decompressed if it is compressed. +should be decompressed if it is compressed (where the compression +format is inferred). (See \scheme{open-output-file}.) The default is \scheme{uncompressed}, so the \scheme{uncompressed} option is useful only as documentation. @@ -1828,12 +1858,7 @@ The default behavior is to raise an exception. The mutually exclusive \scheme{compressed} and \scheme{uncompressed} options determine whether the output file is to be compressed. -Compression is performed with the use of the -zlib compression library developed by -Jean-loup Gailly and Mark Adler. -It is therefore compatible with the \scheme{gzip} program, which -means that \scheme{gzip} may be used to uncompress files produced -by {\ChezScheme} and visa versa. +The compression format is determined by the \scheme{compress-format} parameter. Files are uncompressed by default, so the \scheme{uncompressed} option is useful only as documentation. diff --git a/csug/objects.stex b/csug/objects.stex index 06776c2852..f94ddd3efd 100644 --- a/csug/objects.stex +++ b/csug/objects.stex @@ -1178,7 +1178,8 @@ is immutable; otherwise, the result is an immutable bytevector with the same con The result is the raw compressed data with a minimal header to record the uncompressed size and the compression mode. The result does not include the header that is written by port-based compression using the -\scheme{compressed} option. +\scheme{compressed} option. The compression format is determined by the +\scheme{compress-format} parameter. %---------------------------------------------------------------------------- diff --git a/lz4 b/lz4 new file mode 160000 index 0000000000..c438548312 --- /dev/null +++ b/lz4 @@ -0,0 +1 @@ +Subproject commit c4385483121578644db7f865d4051713b4b254c2 diff --git a/mats/io.ms b/mats/io.ms index cbfb9ec9c6..43897ba8d8 100644 --- a/mats/io.ms +++ b/mats/io.ms @@ -2127,6 +2127,7 @@ ) (mat compression + (parameters [compress-format 'gzip] [compress-format 'lz4]) (let () (define cp (lambda (src dst) @@ -3071,6 +3072,7 @@ ) (mat compression-textual + (parameters [compress-format 'gzip] [compress-format 'lz4]) (let () (define cp (lambda (src dst) diff --git a/mats/mat.ss b/mats/mat.ss index bef5de3b9e..cc04036f81 100644 --- a/mats/mat.ss +++ b/mats/mat.ss @@ -19,7 +19,13 @@ (eval-when (load eval) (define-syntax mat (lambda (x) - (syntax-case x () + (syntax-case x (parameters) + [(_ x (parameters [param val] ...) e ...) + #'(for-each (lambda (p v) + (parameterize ([p v]) + (mat x e ...))) + (list param ...) + (list val ...))] [(_ x e ...) (with-syntax ([(source ...) (map (lambda (clause) diff --git a/release_notes/release_notes.stex b/release_notes/release_notes.stex index ab54a07326..a8c092f896 100644 --- a/release_notes/release_notes.stex +++ b/release_notes/release_notes.stex @@ -58,6 +58,17 @@ Online versions of both books can be found at %----------------------------------------------------------------------------- \section{Functionality Changes}\label{section:functionality} +\subsection{Compression format (9.5.1)} + +The default format for compressed-file writing is now LZ4, while {\tt +gzip} is still supported and can be enabled by setting +\scheme{compress-format} to \scheme{'gzip}. Reading in compressed mode +infers the format, so reading {\tt gzip}-compressed files will still +work without changing \scheme{compress-format}. Reading LZ4-format +files tends to be much faster than reading {\tt gzip}-format files, in +most cases nearly eliminating the load-time cost of compressing +compiled files. + \subsection{Ftype guardians (9.5.1)} Applications that manage memory outside the Scheme heap can leverage diff --git a/s/back.ss b/s/back.ss index c932edd766..3c02c6f63b 100644 --- a/s/back.ss +++ b/s/back.ss @@ -155,6 +155,14 @@ (unless (procedure? x) ($oops who "~s is not a procedure" x)) x))) +(define-who compress-format + ($make-thread-parameter + 'lz4 + (lambda (x) + (unless (or (eq? x 'lz4) (eq? x 'gzip)) + ($oops who "~s is not a supported format" x)) + x))) + (define-who debug-level ($make-thread-parameter 1 diff --git a/s/bytevector.ss b/s/bytevector.ss index 24b6bf388c..724e7e7e2a 100644 --- a/s/bytevector.ss +++ b/s/bytevector.ss @@ -1454,24 +1454,25 @@ ) (let () - ;; Store uncompressed size as u64: + ;; Store uncompressed size as u64, using high bit to indicate LZ4: (define uncompressed-length-length (ftype-sizeof integer-64)) ;; Always big-endian, so that compressed data is portable. ;; It might be useful somehow that valid compressed data always starts - ;; with a 0 byte; otherwise, the expected size would be unrealistically big. + ;; with a 0 or 128 byte; otherwise, the expected size would be unrealistically big. (define uncompressed-length-endianness (endianness big)) (define $bytevector-compress-size - (foreign-procedure "(cs)bytevector_compress_size" (iptr) uptr)) + (foreign-procedure "(cs)bytevector_compress_size" (iptr boolean) uptr)) (define $bytevector-compress - (foreign-procedure "(cs)bytevector_compress" (scheme-object iptr iptr scheme-object iptr iptr) scheme-object)) + (foreign-procedure "(cs)bytevector_compress" (scheme-object iptr iptr scheme-object iptr iptr boolean) scheme-object)) (define $bytevector-uncompress - (foreign-procedure "(cs)bytevector_uncompress" (scheme-object iptr iptr scheme-object iptr iptr) scheme-object)) + (foreign-procedure "(cs)bytevector_uncompress" (scheme-object iptr iptr scheme-object iptr iptr boolean) scheme-object)) (set-who! bytevector-compress (lambda (bv) (unless (bytevector? bv) (not-a-bytevector who bv)) - (let* ([dest-max-len ($bytevector-compress-size (bytevector-length bv))] + (let* ([as-gz? (eq? 'gzip (compress-format))] + [dest-max-len ($bytevector-compress-size (bytevector-length bv) as-gz?)] [dest-alloc-len (min (+ dest-max-len uncompressed-length-length) ;; In the unlikely event of a non-fixnum requested size... (constant maximum-bytevector-length))] @@ -1481,12 +1482,14 @@ (fx- dest-alloc-len uncompressed-length-length) bv 0 - (bytevector-length bv))]) + (bytevector-length bv) + as-gz?)]) (cond [(string? r) ($oops who r bv)] [else ($bytevector-u64-set! dest-bv 0 (bytevector-length bv) uncompressed-length-endianness who) + (unless as-gz? (bytevector-u8-set! dest-bv 0 128)) ; set high bit for LZ4 (bytevector-truncate! dest-bv (fx+ r uncompressed-length-length))]))))) (set-who! bytevector-uncompress @@ -1494,7 +1497,20 @@ (unless (bytevector? bv) (not-a-bytevector who bv)) (unless (>= (bytevector-length bv) uncompressed-length-length) ($oops who "invalid data in source bytevector ~s" bv)) - (let ([dest-length ($bytevector-u64-ref bv 0 uncompressed-length-endianness who)]) + (let* ([as-gz? (not (fx= 128 (bytevector-u8-ref bv 0)))] + [dest-length (cond + [as-gz? + ($bytevector-u64-ref bv 0 uncompressed-length-endianness who)] + ;; Need to skip high bit; likely can skip first 4 bytes + [(and (fx= 0 (bytevector-u8-ref bv 1)) + (fx= 0 (bytevector-u8-ref bv 2)) + (fx= 0 (bytevector-u8-ref bv 3))) + ($bytevector-u32-ref bv 4 uncompressed-length-endianness who)] + [else + ;; Clear high bit the hard way + (+ ($bytevector-u32-ref bv 4 uncompressed-length-endianness who) + (let ([v ($bytevector-u32-ref bv 0 uncompressed-length-endianness who)]) + ((bitwise-arithmetic-shift-left (- v #x80000000) 32))))])]) (unless (and (fixnum? dest-length) ($fxu< dest-length (constant maximum-bytevector-length))) ($oops who "bytevector ~s claims invalid uncompressed size ~s" bv dest-length)) @@ -1504,7 +1520,8 @@ dest-length bv uncompressed-length-length - (fx- (bytevector-length bv) uncompressed-length-length))]) + (fx- (bytevector-length bv) uncompressed-length-length) + as-gz?)]) (cond [(string? r) ($oops who r bv)] [(fx= r dest-length) dest-bv] diff --git a/s/io.ss b/s/io.ss index e3a661ab6b..045db4f5d4 100644 --- a/s/io.ss +++ b/s/io.ss @@ -264,7 +264,7 @@ implementation notes: (foreign-procedure "(cs)new_open_output_fd" (string int boolean boolean boolean - boolean boolean boolean boolean) + boolean boolean boolean boolean boolean) scheme-object)) (define $open-input/output-fd (foreign-procedure "(cs)new_open_input_output_fd" @@ -310,7 +310,7 @@ implementation notes: (define $compress-input-fd (foreign-procedure "(cs)compress_input_fd" (int integer-64) scheme-object)) (define $compress-output-fd - (foreign-procedure "(cs)compress_output_fd" (int) scheme-object)) + (foreign-procedure "(cs)compress_output_fd" (int boolean) scheme-object)) (module (clear-open-files register-open-file registered-open-file? unregister-open-file) (define open-files #f) (define file-guardian) @@ -3185,7 +3185,7 @@ implementation notes: ; reposition to 'unread' any compressed data in the input buffer (set-port-position! p fp) ($compress-input-fd fd fp)) - ($compress-output-fd fd))]) + ($compress-output-fd fd (eq? (compress-format) 'gzip)))]) (when (string? gzfd) ($oops who "failed for ~s: ~(~a~)" p gzfd)) (unless (eqv? gzfd fd) ; uncompressed input port (assert (box? gzfd)) @@ -4091,7 +4091,8 @@ implementation notes: (let ([fd (critical-section ($open-output-fd filename perms no-create no-fail no-truncate - append lock replace compressed))]) + append lock replace compressed + (and compressed (eq? (compress-format) 'gzip))))]) (when (pair? fd) (open-oops who filename options fd)) (open-binary-fd-output-port who filename fd #t b-mode lock compressed))))) diff --git a/s/primdata.ss b/s/primdata.ss index 34fbad63a1..cd25628b36 100644 --- a/s/primdata.ss +++ b/s/primdata.ss @@ -925,6 +925,7 @@ (compile-library-handler [sig [() -> (procedure)] [(procedure) -> (void)]] [flags]) (compile-profile [sig [() -> (ptr)] [(ptr) -> (void)]] [flags unrestricted]) (compile-program-handler [sig [() -> (procedure)] [(procedure) -> (void)]] [flags]) + (compress-format [sig [() -> (symbol)] [(symbol) -> (void)]] [flags]) (console-error-port [sig [() -> (textual-output-port)] [(textual-output-port) -> (void)]] [flags]) (console-input-port [sig [() -> (textual-input-port)] [(textual-input-port) -> (void)]] [flags]) (console-output-port [sig [() -> (textual-output-port)] [(textual-output-port) -> (void)]] [flags]) diff --git a/workarea b/workarea index 995fd21aec..d67ce7d2cb 100755 --- a/workarea +++ b/workarea @@ -172,6 +172,12 @@ for dir in `echo zlib` ; do fi done +for dir in `echo lz4` ; do + if [ ! -e $W/$dir ] ; then + /bin/cp -R $dir $W/$dir + fi +done + workdir $W/boot workdir $W/boot/$M (cd $W/boot/$M; workln ../../../boot/$M/scheme.h scheme.h)