From dceb2d68236ca7f58c43deec1fd6f498b3612339 Mon Sep 17 00:00:00 2001 From: Suzanne Soy Date: Sun, 21 Feb 2021 22:36:59 +0000 Subject: [PATCH] gitlab's stdout and stderr are not in sync (?) --- hash-files.py | 25 ++++++++++++++----------- test.py | 24 ++++++++++++------------ 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/hash-files.py b/hash-files.py index 812e218..ad46ea9 100755 --- a/hash-files.py +++ b/hash-files.py @@ -4,14 +4,18 @@ import os import sys import subprocess +def debug(s): + print(s, file=sys.stderr, flush=True) + + def hashFile(filename): result = subprocess.check_output(['sha256sum', '--binary', '--zero', filename])[0:64] - print("hashFile("+filename+") = "+str(result), file=sys.stderr) + debug("hashFile("+filename+") = "+str(result)) return result def hash1(bytes_): result = subprocess.check_output(['sha256sum', '--binary', '--zero'], input=bytes_)[0:64] - print("hash1("+str(bytes_)+") = "+str(result), file=sys.stderr) + debug("hash1("+str(bytes_)+") = "+str(result)) return result # @@ -38,12 +42,12 @@ def hashGit(path): FETCH_HEAD = "FETCH_HEAD" if ref_exists(path, "FETCH_HEAD") else '' HEAD = "HEAD" if ref_exists(path, "HEAD") else '' result = subprocess.check_output(['sh', '-c', git_command.format(HEAD=HEAD, FETCH_HEAD=FETCH_HEAD)], cwd=path) - print("hashGit("+path+") = "+str(result), file=sys.stderr) + debug("hashGit("+path+") = "+str(result)) return result def hashSqlite3(path): result= subprocess.check_output(['sh', '-c', 'sqlite3 "$1" .dump | sort | sha256sum --binary --zero', '--', os.path.abspath(path)]) - print("hashSqlite3("+path+") = "+str(result), file=sys.stderr) + debug("hashSqlite3("+path+") = "+str(result)) return result def ignore_exitcode(cmd, **kwargs): @@ -59,27 +63,26 @@ def is_git(x): # TODO: if a file which is inside a larger git dir is passed on the CLI, this still returns True :-( def recur(depth, x): - #print(x) # initial list of paths if isinstance(x, list): - print("ROOT " + str(depth) + ' [' + ', '.join(x) + ']', file=sys.stderr) + debug("ROOT " + str(depth) + ' [' + ', '.join(x) + ']') return b'root\0' + b''.join(recur(depth + 1, os.path.abspath(path)) + b' ' + path.encode('utf-8') + b'\0' for path in sorted(x)) # GIT repo elif is_git(x): - print("GIT DIR " + str(depth) + ' ' + x, file=sys.stderr) + debug("GIT DIR " + str(depth) + ' ' + x) return hash1(b'git-versioned folder\0' + hashGit(x)) # directory elif os.path.isdir(x): - print("DIR " + str(depth) + ' ' + x, file=sys.stderr) + debug("DIR " + str(depth) + ' ' + x) return hash1(b'directory\0' + b''.join(recur(depth + 1, os.path.join(x, entry)) + b' ' + entry.encode('utf-8') + b'\0' for entry in os.listdir(x))) elif b'SQLite 3.x database' in subprocess.check_output(["file", x]): - print("SQLITE3 " + str(depth) + ' ' + x, file=sys.stderr) + debug("SQLITE3 " + str(depth) + ' ' + x) return hashSqlite3(x) # Just a file elif os.path.isfile(x): - print("PLAIN FILE " + str(depth) + ' ' + x, file=sys.stderr) + debug("PLAIN FILE " + str(depth) + ' ' + x) return hashFile(x) else: sys.exit("unknown file type for %s" % os.path.abspath(x)) -print(hash1(recur(0, sys.argv[1:])).decode('utf-8')) +print(hash1(recur(0, sys.argv[1:])).decode('utf-8'), flush=True) diff --git a/test.py b/test.py index 52a3a0a..60b6e18 100755 --- a/test.py +++ b/test.py @@ -8,9 +8,9 @@ with tempfile.TemporaryDirectory(prefix="test", dir="/tmp") as tempdir: os.mkdir(tempdir+'/test/foo') h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip() if h == b'dc99f8161ccf245e178102a00264e4f4f43cd0048ea525b6c9e226777414352f': - print("test passed: empty") + print("test passed: empty", flush=True) else: - print("TEST FAILED: empty: got hash " + repr(h)) + print("TEST FAILED: empty: got hash " + repr(h), flush=True) exit(1) # Plain text file @@ -20,9 +20,9 @@ with tempfile.TemporaryDirectory(prefix="test", dir="/tmp") as tempdir: os.system('echo a > '+tempdir+'/test/foo/x') h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip() if h == b'6b393b2233479ccc54975f83f4de0d39592d5ab78cd02b19597e7bbe97f43cf1': - print("test passed: plain text file") + print("test passed: plain text file", flush=True) else: - print("TEST FAILED: plain text file: got hash " + repr(h)) + print("TEST FAILED: plain text file: got hash " + repr(h), flush=True) exit(1) # Plain text file and empty folder in subdirectory @@ -35,9 +35,9 @@ with tempfile.TemporaryDirectory(prefix="test", dir="/tmp") as tempdir: os.system('echo a > '+tempdir+'/test/foo/baz/titi') h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip() if h == b'7421373b28f6a1929228a3bd7ecb23123d25da36c9bbe41518c7a6252f351712': - print("test passed: plain text and empty folder in subdirectory") + print("test passed: plain text and empty folder in subdirectory", flush=True) else: - print("TEST FAILED: plain text and empty folder in subdirectory: got hash " + repr(h)) + print("TEST FAILED: plain text and empty folder in subdirectory: got hash " + repr(h), flush=True) exit(1) # Git directories @@ -58,10 +58,10 @@ with tempfile.TemporaryDirectory(prefix="test", dir="/tmp") as tempdir: os.system('echo a > '+tempdir+'/test/foo/baz/titi') h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip() if h == b'8a84206ece36f07d2c408e565ec506bab407d6e1c645eb4a5c7d057049956110': - print("test passed: git") + print("test passed: git", flush=True) subprocess.check_output("tar -zcf /tmp/debug-git.tar.gz .", cwd=tempdir, shell=True) else: - print("TEST FAILED: git: got hash " + repr(h)) + print("TEST FAILED: git: got hash " + repr(h), flush=True) subprocess.check_output("tar -zcf /tmp/debug-git.tar.gz .", cwd=tempdir, shell=True) exit(0) @@ -95,10 +95,10 @@ with tempfile.TemporaryDirectory(prefix="test", dir="/tmp") as tempdir: os.system('echo a > '+tempdir+'/test/foo/baz/titi') h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip() if h == b'c04f602dcb7eec19433c12981f58d653c61ac7453da60e375f2f5587dc57f474': - print("test passed: sqlite") + print("test passed: sqlite", flush=True) subprocess.check_output("tar -zcf /tmp/debug-sql.tar.gz .", cwd=tempdir, shell=True) else: - print("TEST FAILED: sqlite got hash " + repr(h)) + print("TEST FAILED: sqlite got hash " + repr(h), flush=True) subprocess.check_output("tar -zcf /tmp/debug-sql.tar.gz .", cwd=tempdir, shell=True) exit(0) @@ -132,9 +132,9 @@ with tempfile.TemporaryDirectory(prefix="test", dir="/tmp") as tempdir: os.system('echo a > '+tempdir+'/test/foo/baz/titi') h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip() if h == b'b0aae011d0e438a78d6e2652478ec667b7650b9c29dc6d7235ccce26a75c95cb': - print("test passed: sqlite big table") + print("test passed: sqlite big table", flush=True) subprocess.check_output("tar -zcf /tmp/debug-sql.tar.gz .", cwd=tempdir, shell=True) else: - print("TEST FAILED: sqlite big table got hash " + repr(h)) + print("TEST FAILED: sqlite big table got hash " + repr(h), flush=True) subprocess.check_output("tar -zcf /tmp/debug-sql.tar.gz .", cwd=tempdir, shell=True) exit(0) \ No newline at end of file