suzanne.soy/nixpkgs
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

copy-tarballs: Cache S3 lookups

Browse Source
This commit is contained in:
Eelco Dolstra 2015-12-16 14:17:33 +01:00
parent 7cd3d502bb
commit d5371eb029
1 changed files with 17 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
maintainers/scripts/copy-tarballs.pl
View File

@ -1,5 +1,5 @@
#! /usr/bin/env nix-shell #! /usr/bin/env nix-shell
#! nix-shell -i perl -p perl perlPackages.NetAmazonS3 nixUnstable #! nix-shell -i perl -p perl perlPackages.NetAmazonS3 perlPackages.FileSlurp nixUnstable
# This command uploads tarballs to tarballs.nixos.org, the # This command uploads tarballs to tarballs.nixos.org, the
# content-addressed cache used by fetchurl as a fallback for when # content-addressed cache used by fetchurl as a fallback for when
@ -17,6 +17,7 @@ use strict;
use warnings; use warnings;
use File::Basename; use File::Basename;
use File::Path; use File::Path;
use File::Slurp;
use JSON; use JSON;
use Net::Amazon::S3; use Net::Amazon::S3;
use Nix::Store; use Nix::Store;
@ -33,9 +34,21 @@ my $s3 = Net::Amazon::S3->new(
my $bucket = $s3->bucket("nixpkgs-tarballs") or die; my $bucket = $s3->bucket("nixpkgs-tarballs") or die;
my $cacheFile = "/tmp/copy-tarballs-cache";
my %cache;
$cache{$_} = 1 foreach read_file($cacheFile, err_mode => 'quiet', chomp => 1);
END() {
write_file($cacheFile, map { "$_\n" } keys %cache);
}
sub alreadyMirrored { sub alreadyMirrored {
my ($algo, $hash) = @_; my ($algo, $hash) = @_;
return defined $bucket->get_key("$algo/$hash"); my $key = "$algo/$hash";
return 1 if defined $cache{$key};
my $res = defined $bucket->get_key($key);
$cache{$key} = 1 if $res;
return $res;
} }
sub uploadFile { sub uploadFile {
@ -54,6 +67,7 @@ sub uploadFile {
print STDERR "uploading $fn to $mainKey...\n"; print STDERR "uploading $fn to $mainKey...\n";
$bucket->add_key_filename($mainKey, $fn, { 'x-amz-meta-original-name' => $name }) $bucket->add_key_filename($mainKey, $fn, { 'x-amz-meta-original-name' => $name })
or die "failed to upload $fn to $mainKey\n"; or die "failed to upload $fn to $mainKey\n";
$cache{$mainKey} = 1;
# Create redirects from the other hash types. # Create redirects from the other hash types.
sub redirect { sub redirect {
@ -61,6 +75,7 @@ sub uploadFile {
#print STDERR "linking $name to $dest...\n"; #print STDERR "linking $name to $dest...\n";
$bucket->add_key($name, "", { 'x-amz-website-redirect-location' => "/" . $dest }) $bucket->add_key($name, "", { 'x-amz-website-redirect-location' => "/" . $dest })
or die "failed to create redirect from $name to $dest\n"; or die "failed to create redirect from $name to $dest\n";
$cache{$name} = 1;
} }
redirect "md5/$md5_16", $mainKey; redirect "md5/$md5_16", $mainKey;
redirect "sha1/$sha1_16", $mainKey; redirect "sha1/$sha1_16", $mainKey;