buildGoModule: function for packaging Go modules

The function buildGoModule builds Go programs managed with Go modules. It builds
a Go module through a two phase build:

- An intermediate fetcher derivation. This derivation will be used to
  fetch all of the dependencies of the Go module.
- A final derivation will use the output of the intermediate derivation
  to build the binaries and produce the final output.

doc/languages-frameworks/go.xml

@@ -3,8 +3,87 @@
          xml:id="sec-language-go">
  <title>Go</title>
 
+ <section xml:id="ssec-go-modules">
+  <title>Go modules</title>
+
   <para>
-  The function <varname>buildGoPackage</varname> builds standard Go programs.
+   The function <varname> buildGoModule </varname> builds Go programs managed
+   with Go modules. It builds a
+   <link xlink:href="https://github.com/golang/go/wiki/Modules">Go
+   modules</link> through a two phase build:
+   <itemizedlist>
+    <listitem>
+     <para>
+      An intermediate fetcher derivation. This derivation will be used to fetch
+      all of the dependencies of the Go module.
+     </para>
+    </listitem>
+    <listitem>
+     <para>
+      A final derivation will use the output of the intermediate derivation to
+      build the binaries and produce the final output.
+     </para>
+    </listitem>
+   </itemizedlist>
+  </para>
+
+  <example xml:id='ex-buildGoModule'>
+   <title>buildGoModule</title>
+<programlisting>
+pet = buildGoModule rec {
+  name = "pet-${version}";
+  version = "0.3.4";
+
+  src = fetchFromGitHub {
+    owner = "knqyf263";
+    repo = "pet";
+    rev = "v${version}";
+    sha256 = "0m2fzpqxk7hrbxsgqplkg7h2p7gv6s1miymv3gvw0cz039skag0s";
+  };
+
+  modSha256 = "1879j77k96684wi554rkjxydrj8g3hpp0kvxz03sd8dmwr3lh83j"; <co xml:id='ex-buildGoModule-1' />
+
+  subPackages = [ "." ]; <co xml:id='ex-buildGoModule-2' />
+
+  meta = with lib; {
+    description = "Simple command-line snippet manager, written in Go";
+    homepage = https://github.com/knqyf263/pet;
+    license = licenses.mit;
+    maintainers = with maintainers; [ kalbasit ];
+    platforms = platforms.linux ++ platforms.darwin;
+  };
+}
+</programlisting>
+  </example>
+
+  <para>
+   <xref linkend='ex-buildGoModule'/> is an example expression using
+   buildGoModule, the following arguments are of special significance to the
+   function:
+   <calloutlist>
+    <callout arearefs='ex-buildGoModule-1'>
+     <para>
+      <varname>modSha256</varname> is the hash of the output of the
+      intermediate fetcher derivation.
+     </para>
+    </callout>
+    <callout arearefs='ex-buildGoModule-2'>
+     <para>
+      <varname>subPackages</varname> limits the builder from building child
+      packages that have not been listed. If <varname>subPackages</varname> is
+      not specified, all child packages will be built.
+     </para>
+    </callout>
+   </calloutlist>
+  </para>
+ </section>
+
+ <section xml:id="ssec-go-legacy">
+  <title>Go legacy</title>
+
+  <para>
+   The function <varname> buildGoPackage </varname> builds legacy Go programs,
+   not supporting Go modules.
   </para>
 
   <example xml:id='ex-buildGoPackage'>
@@ -55,9 +134,9 @@ deis = buildGoPackage rec {
     </callout>
     <callout arearefs='ex-buildGoPackage-3'>
      <para>
-     <varname>goDeps</varname> is where the Go dependencies of a Go program are
+      <varname>goDeps</varname> is where the Go dependencies of a Go program
-     listed as a list of package source identified by Go import path. It could
+      are listed as a list of package source identified by Go import path. It
-     be imported as a separate <varname>deps.nix</varname> file for
+      could be imported as a separate <varname>deps.nix</varname> file for
       readability. The dependency data structure is described below.
      </para>
     </callout>
@@ -117,9 +196,10 @@ deis = buildGoPackage rec {
     </callout>
     <callout arearefs='ex-goDeps-3'>
      <para>
-     <varname>fetch type</varname> that needs to be used to get package source.
+      <varname>fetch type</varname> that needs to be used to get package
-     If <varname>git</varname> is used there should be <varname>url</varname>,
+      source. If <varname>git</varname> is used there should be
-     <varname>rev</varname> and <varname>sha256</varname> defined next to it.
+      <varname>url</varname>, <varname>rev</varname> and
+      <varname>sha256</varname> defined next to it.
      </para>
     </callout>
    </calloutlist>
@@ -158,3 +238,4 @@ done
 </screen>
   </para>
  </section>
+</section>

pkgs/development/go-modules/generic/default.nix

@@ -0,0 +1,214 @@
+{ go, cacert, git, lib, removeReferencesTo, stdenv }:
+
+{ name
+, src
+, buildInputs ? []
+, nativeBuildInputs ? []
+, passthru ? {}
+, patches ? []
+
+# modSha256 is the sha256 of the vendored dependencies
+, modSha256
+
+# We want parallel builds by default
+, enableParallelBuilding ? true
+
+# Disabled flag
+, disabled ? false
+
+# Do not enable this without good reason
+# IE: programs coupled with the compiler
+, allowGoReference ? false
+
+, meta ? {}
+
+, ... }@args':
+
+with builtins;
+
+let
+  args = removeAttrs args' [ "modSha256" "disabled" ];
+
+  removeReferences = [ ] ++ lib.optional (!allowGoReference) go;
+
+  removeExpr = refs: ''remove-references-to ${lib.concatMapStrings (ref: " -t ${ref}") refs}'';
+
+  go-modules = go.stdenv.mkDerivation {
+    name = "${name}-go-modules";
+
+    nativeBuildInputs = [ go git ];
+
+    inherit (args) src;
+    inherit (go) GOOS GOARCH;
+
+    patches = args.patches or [];
+
+    GO111MODULE = "on";
+
+    # XXX: Add support for other fetchers, such as hg, bzr and alike.
+    GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+
+    # Instruct Go where to find the cacert.
+    # SSL_CERT_FILE is used by Linux machines.
+    # NIX_SSL_CERT_FILE is used by Darwin machines based on
+    # pkgs/development/compilers/go/ssl-cert-file-1.9.patch.
+    NIX_SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+    SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+
+    impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [
+      "GIT_PROXY_COMMAND" "SOCKS_SERVER"
+    ];
+
+    configurePhase = args.modConfigurePhase or ''
+      runHook preConfigure
+
+      export GOCACHE=$TMPDIR/go-cache
+      export GOPATH="$TMPDIR/go"
+
+      runHook postConfigure
+    '';
+
+    buildPhase = args.modBuildPhase or ''
+      runHook preBuild
+
+      go mod download
+
+      runHook postBuild
+    '';
+
+    installPhase = args.modInstallPhase or ''
+      runHook preInstall
+
+      cp -r "''${GOPATH}/pkg/mod/cache/download" $out
+
+      runHook postInstall
+    '';
+
+    dontFixup = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+    outputHash = modSha256;
+  };
+
+  package = go.stdenv.mkDerivation (args // {
+    nativeBuildInputs = [ removeReferencesTo go ] ++ nativeBuildInputs;
+
+    inherit (go) GOOS GOARCH;
+
+    GO111MODULE = "on";
+
+    configurePhase = args.configurePhase or ''
+      runHook preConfigure
+
+      export GOCACHE=$TMPDIR/go-cache
+      export GOPATH="$TMPDIR/go"
+      export GOPROXY=file://${go-modules}
+
+      runHook postConfigure
+    '';
+
+    buildPhase = args.buildPhase or ''
+      runHook preBuild
+
+      buildGoDir() {
+        local d; local cmd;
+        cmd="$1"
+        d="$2"
+        . $TMPDIR/buildFlagsArray
+        echo "$d" | grep -q "\(/_\|examples\|Godeps\|testdata\)" && return 0
+        [ -n "$excludedPackages" ] && echo "$d" | grep -q "$excludedPackages" && return 0
+        local OUT
+        if ! OUT="$(go $cmd $buildFlags "''${buildFlagsArray[@]}" -v -p $NIX_BUILD_CORES $d 2>&1)"; then
+          if ! echo "$OUT" | grep -qE '(no( buildable| non-test)?|build constraints exclude all) Go (source )?files'; then
+            echo "$OUT" >&2
+            return 1
+          fi
+        fi
+        if [ -n "$OUT" ]; then
+          echo "$OUT" >&2
+        fi
+        return 0
+      }
+
+      getGoDirs() {
+        local type;
+        type="$1"
+        if [ -n "$subPackages" ]; then
+          echo "./$subPackages"
+        else
+          find . -type f -name \*$type.go -exec dirname {} \; | grep -v "/vendor/" | sort --unique
+        fi
+      }
+
+      if (( "''${NIX_DEBUG:-0}" >= 1 )); then
+        buildFlagsArray+=(-x)
+      fi
+
+      if [ ''${#buildFlagsArray[@]} -ne 0 ]; then
+        declare -p buildFlagsArray > $TMPDIR/buildFlagsArray
+      else
+        touch $TMPDIR/buildFlagsArray
+      fi
+      if [ -z "$enableParallelBuilding" ]; then
+          export NIX_BUILD_CORES=1
+      fi
+      for pkg in $(getGoDirs ""); do
+        buildGoDir install "$pkg"
+      done
+    '' + lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''
+      # normalize cross-compiled builds w.r.t. native builds
+      (
+        dir=$GOPATH/bin/${go.GOOS}_${go.GOARCH}
+        if [[ -n "$(shopt -s nullglob; echo $dir/*)" ]]; then
+          mv $dir/* $dir/..
+        fi
+        if [[ -d $dir ]]; then
+          rmdir $dir
+        fi
+      )
+    '' + ''
+      runHook postBuild
+    '';
+
+    doCheck = args.doCheck or false;
+    checkPhase = args.checkPhase or ''
+      runHook preCheck
+
+      for pkg in $(getGoDirs test); do
+        buildGoDir test "$pkg"
+      done
+
+      runHook postCheck
+    '';
+
+    installPhase = args.installPhase or ''
+      runHook preInstall
+
+      mkdir -p $out
+      dir="$GOPATH/bin"
+      [ -e "$dir" ] && cp -r $dir $out
+
+      runHook postInstall
+    '';
+
+    preFixup = (args.preFixup or "") + ''
+      find $out/bin -type f -exec ${removeExpr removeReferences} '{}' + || true
+    '';
+
+    disallowedReferences = lib.optional (!allowGoReference) go;
+
+    passthru = passthru // { inherit go go-modules; };
+
+    meta = {
+      # Add default meta information
+      platforms = go.meta.platforms or lib.platforms.all;
+    } // meta // {
+      # add an extra maintainer to every package
+      maintainers = (meta.maintainers or []) ++
+                    [ lib.maintainers.kalbasit ];
+    };
+  });
+in if disabled then
+  throw "${package.name} not supported for go ${go.meta.branch}"
+else
+  package

pkgs/development/go-modules/tools/setup-hook.sh

@@ -0,0 +1 @@
+export GOTOOLDIR=@bin@/bin

pkgs/top-level/all-packages.nix

@@ -13351,6 +13351,12 @@ in
 
   buildGoPackage = buildGo111Package;
 
+  buildGo111Module = callPackage ../development/go-modules/generic {
+    go = buildPackages.go_1_11;
+  };
+
+  buildGoModule = buildGo111Module;
+
   go2nix = callPackage ../development/tools/go2nix { };
 
   leaps = callPackage ../development/tools/leaps { };