suzanne.soy/hash-files
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

gitlab's stdout and stderr are not in sync (?)

Browse Source
This commit is contained in:
Suzanne Soy 2021-02-21 22:36:59 +00:00
parent df5d372b7e
commit dceb2d6823
2 changed files with 26 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
hash-files.py
View File

@ -4,14 +4,18 @@ import os
import sys import sys
import subprocess import subprocess
def debug(s):
print(s, file=sys.stderr, flush=True)
def hashFile(filename): def hashFile(filename):
result = subprocess.check_output(['sha256sum', '--binary', '--zero', filename])[0:64] result = subprocess.check_output(['sha256sum', '--binary', '--zero', filename])[0:64]
print("hashFile("+filename+") = "+str(result), file=sys.stderr) debug("hashFile("+filename+") = "+str(result))
return result return result
def hash1(bytes_): def hash1(bytes_):
result = subprocess.check_output(['sha256sum', '--binary', '--zero'], input=bytes_)[0:64] result = subprocess.check_output(['sha256sum', '--binary', '--zero'], input=bytes_)[0:64]
print("hash1("+str(bytes_)+") = "+str(result), file=sys.stderr) debug("hash1("+str(bytes_)+") = "+str(result))
return result return result
# #
@ -38,12 +42,12 @@ def hashGit(path):
FETCH_HEAD = "FETCH_HEAD" if ref_exists(path, "FETCH_HEAD") else '' FETCH_HEAD = "FETCH_HEAD" if ref_exists(path, "FETCH_HEAD") else ''
HEAD = "HEAD" if ref_exists(path, "HEAD") else '' HEAD = "HEAD" if ref_exists(path, "HEAD") else ''
result = subprocess.check_output(['sh', '-c', git_command.format(HEAD=HEAD, FETCH_HEAD=FETCH_HEAD)], cwd=path) result = subprocess.check_output(['sh', '-c', git_command.format(HEAD=HEAD, FETCH_HEAD=FETCH_HEAD)], cwd=path)
print("hashGit("+path+") = "+str(result), file=sys.stderr) debug("hashGit("+path+") = "+str(result))
return result return result
def hashSqlite3(path): def hashSqlite3(path):
result= subprocess.check_output(['sh', '-c', 'sqlite3 "$1" .dump | sort | sha256sum --binary --zero', '--', os.path.abspath(path)]) result= subprocess.check_output(['sh', '-c', 'sqlite3 "$1" .dump | sort | sha256sum --binary --zero', '--', os.path.abspath(path)])
print("hashSqlite3("+path+") = "+str(result), file=sys.stderr) debug("hashSqlite3("+path+") = "+str(result))
return result return result
def ignore_exitcode(cmd, **kwargs): def ignore_exitcode(cmd, **kwargs):
@ -59,27 +63,26 @@ def is_git(x):
# TODO: if a file which is inside a larger git dir is passed on the CLI, this still returns True :-( # TODO: if a file which is inside a larger git dir is passed on the CLI, this still returns True :-(
def recur(depth, x): def recur(depth, x):
#print(x)
# initial list of paths # initial list of paths
if isinstance(x, list): if isinstance(x, list):
print("ROOT " + str(depth) + ' [' + ', '.join(x) + ']', file=sys.stderr) debug("ROOT " + str(depth) + ' [' + ', '.join(x) + ']')
return b'root\0' + b''.join(recur(depth + 1, os.path.abspath(path)) + b' ' + path.encode('utf-8') + b'\0' for path in sorted(x)) return b'root\0' + b''.join(recur(depth + 1, os.path.abspath(path)) + b' ' + path.encode('utf-8') + b'\0' for path in sorted(x))
# GIT repo # GIT repo
elif is_git(x): elif is_git(x):
print("GIT DIR " + str(depth) + ' ' + x, file=sys.stderr) debug("GIT DIR " + str(depth) + ' ' + x)
return hash1(b'git-versioned folder\0' + hashGit(x)) return hash1(b'git-versioned folder\0' + hashGit(x))
# directory # directory
elif os.path.isdir(x): elif os.path.isdir(x):
print("DIR " + str(depth) + ' ' + x, file=sys.stderr) debug("DIR " + str(depth) + ' ' + x)
return hash1(b'directory\0' + b''.join(recur(depth + 1, os.path.join(x, entry)) + b' ' + entry.encode('utf-8') + b'\0' for entry in os.listdir(x))) return hash1(b'directory\0' + b''.join(recur(depth + 1, os.path.join(x, entry)) + b' ' + entry.encode('utf-8') + b'\0' for entry in os.listdir(x)))
elif b'SQLite 3.x database' in subprocess.check_output(["file", x]): elif b'SQLite 3.x database' in subprocess.check_output(["file", x]):
print("SQLITE3 " + str(depth) + ' ' + x, file=sys.stderr) debug("SQLITE3 " + str(depth) + ' ' + x)
return hashSqlite3(x) return hashSqlite3(x)
# Just a file # Just a file
elif os.path.isfile(x): elif os.path.isfile(x):
print("PLAIN FILE " + str(depth) + ' ' + x, file=sys.stderr) debug("PLAIN FILE " + str(depth) + ' ' + x)
return hashFile(x) return hashFile(x)
else: else:
sys.exit("unknown file type for %s" % os.path.abspath(x)) sys.exit("unknown file type for %s" % os.path.abspath(x))
print(hash1(recur(0, sys.argv[1:])).decode('utf-8')) print(hash1(recur(0, sys.argv[1:])).decode('utf-8'), flush=True)

24
test.py
View File

@ -8,9 +8,9 @@ with tempfile.TemporaryDirectory(prefix="test", dir="/tmp") as tempdir:
os.mkdir(tempdir+'/test/foo') os.mkdir(tempdir+'/test/foo')
h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip() h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip()
if h == b'dc99f8161ccf245e178102a00264e4f4f43cd0048ea525b6c9e226777414352f': if h == b'dc99f8161ccf245e178102a00264e4f4f43cd0048ea525b6c9e226777414352f':
print("test passed: empty") print("test passed: empty", flush=True)
else: else:
print("TEST FAILED: empty: got hash " + repr(h)) print("TEST FAILED: empty: got hash " + repr(h), flush=True)
exit(1) exit(1)
# Plain text file # Plain text file
@ -20,9 +20,9 @@ with tempfile.TemporaryDirectory(prefix="test", dir="/tmp") as tempdir:
os.system('echo a > '+tempdir+'/test/foo/x') os.system('echo a > '+tempdir+'/test/foo/x')
h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip() h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip()
if h == b'6b393b2233479ccc54975f83f4de0d39592d5ab78cd02b19597e7bbe97f43cf1': if h == b'6b393b2233479ccc54975f83f4de0d39592d5ab78cd02b19597e7bbe97f43cf1':
print("test passed: plain text file") print("test passed: plain text file", flush=True)
else: else:
print("TEST FAILED: plain text file: got hash " + repr(h)) print("TEST FAILED: plain text file: got hash " + repr(h), flush=True)
exit(1) exit(1)
# Plain text file and empty folder in subdirectory # Plain text file and empty folder in subdirectory
@ -35,9 +35,9 @@ with tempfile.TemporaryDirectory(prefix="test", dir="/tmp") as tempdir:
os.system('echo a > '+tempdir+'/test/foo/baz/titi') os.system('echo a > '+tempdir+'/test/foo/baz/titi')
h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip() h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip()
if h == b'7421373b28f6a1929228a3bd7ecb23123d25da36c9bbe41518c7a6252f351712': if h == b'7421373b28f6a1929228a3bd7ecb23123d25da36c9bbe41518c7a6252f351712':
print("test passed: plain text and empty folder in subdirectory") print("test passed: plain text and empty folder in subdirectory", flush=True)
else: else:
print("TEST FAILED: plain text and empty folder in subdirectory: got hash " + repr(h)) print("TEST FAILED: plain text and empty folder in subdirectory: got hash " + repr(h), flush=True)
exit(1) exit(1)
# Git directories # Git directories
@ -58,10 +58,10 @@ with tempfile.TemporaryDirectory(prefix="test", dir="/tmp") as tempdir:
os.system('echo a > '+tempdir+'/test/foo/baz/titi') os.system('echo a > '+tempdir+'/test/foo/baz/titi')
h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip() h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip()
if h == b'8a84206ece36f07d2c408e565ec506bab407d6e1c645eb4a5c7d057049956110': if h == b'8a84206ece36f07d2c408e565ec506bab407d6e1c645eb4a5c7d057049956110':
print("test passed: git") print("test passed: git", flush=True)
subprocess.check_output("tar -zcf /tmp/debug-git.tar.gz .", cwd=tempdir, shell=True) subprocess.check_output("tar -zcf /tmp/debug-git.tar.gz .", cwd=tempdir, shell=True)
else: else:
print("TEST FAILED: git: got hash " + repr(h)) print("TEST FAILED: git: got hash " + repr(h), flush=True)
subprocess.check_output("tar -zcf /tmp/debug-git.tar.gz .", cwd=tempdir, shell=True) subprocess.check_output("tar -zcf /tmp/debug-git.tar.gz .", cwd=tempdir, shell=True)
exit(0) exit(0)
@ -95,10 +95,10 @@ with tempfile.TemporaryDirectory(prefix="test", dir="/tmp") as tempdir:
os.system('echo a > '+tempdir+'/test/foo/baz/titi') os.system('echo a > '+tempdir+'/test/foo/baz/titi')
h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip() h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip()
if h == b'c04f602dcb7eec19433c12981f58d653c61ac7453da60e375f2f5587dc57f474': if h == b'c04f602dcb7eec19433c12981f58d653c61ac7453da60e375f2f5587dc57f474':
print("test passed: sqlite") print("test passed: sqlite", flush=True)
subprocess.check_output("tar -zcf /tmp/debug-sql.tar.gz .", cwd=tempdir, shell=True) subprocess.check_output("tar -zcf /tmp/debug-sql.tar.gz .", cwd=tempdir, shell=True)
else: else:
print("TEST FAILED: sqlite got hash " + repr(h)) print("TEST FAILED: sqlite got hash " + repr(h), flush=True)
subprocess.check_output("tar -zcf /tmp/debug-sql.tar.gz .", cwd=tempdir, shell=True) subprocess.check_output("tar -zcf /tmp/debug-sql.tar.gz .", cwd=tempdir, shell=True)
exit(0) exit(0)
@ -132,9 +132,9 @@ with tempfile.TemporaryDirectory(prefix="test", dir="/tmp") as tempdir:
os.system('echo a > '+tempdir+'/test/foo/baz/titi') os.system('echo a > '+tempdir+'/test/foo/baz/titi')
h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip() h = subprocess.check_output([os.path.abspath('./hash-files.py'), 'test/foo'], cwd=tempdir).strip()
if h == b'b0aae011d0e438a78d6e2652478ec667b7650b9c29dc6d7235ccce26a75c95cb': if h == b'b0aae011d0e438a78d6e2652478ec667b7650b9c29dc6d7235ccce26a75c95cb':
print("test passed: sqlite big table") print("test passed: sqlite big table", flush=True)
subprocess.check_output("tar -zcf /tmp/debug-sql.tar.gz .", cwd=tempdir, shell=True) subprocess.check_output("tar -zcf /tmp/debug-sql.tar.gz .", cwd=tempdir, shell=True)
else: else:
print("TEST FAILED: sqlite big table got hash " + repr(h)) print("TEST FAILED: sqlite big table got hash " + repr(h), flush=True)
subprocess.check_output("tar -zcf /tmp/debug-sql.tar.gz .", cwd=tempdir, shell=True) subprocess.check_output("tar -zcf /tmp/debug-sql.tar.gz .", cwd=tempdir, shell=True)
exit(0) exit(0)