@ -46,12 +46,11 @@ import (
"encoding/base64" "encoding/base64"
"flag" "flag"
"fmt" "fmt"
"go/parser"
"go/token"
"io/ioutil" "io/ioutil"
"log" "log"
"os" "os"
"os/exec" "os/exec"
"path"
"path/filepath" "path/filepath"
"regexp" "regexp"
"runtime" "runtime"
@ -148,6 +147,11 @@ var (
"golang-1.11" : "/usr/lib/go-1.11" , "golang-1.11" : "/usr/lib/go-1.11" ,
"golang-go" : "/usr/lib/go" , "golang-go" : "/usr/lib/go" ,
} }
// This is the version of go that will be downloaded by
//
// go run ci.go install -dlgo
dlgoVersion = "1.15.4"
) )
var GOBIN , _ = filepath . Abs ( filepath . Join ( "build" , "bin" ) ) var GOBIN , _ = filepath . Abs ( filepath . Join ( "build" , "bin" ) )
@ -198,19 +202,19 @@ func main() {
func doInstall ( cmdline [ ] string ) { func doInstall ( cmdline [ ] string ) {
var ( var (
dlgo = flag . Bool ( "dlgo" , false , "Download Go and build with it" )
arch = flag . String ( "arch" , "" , "Architecture to cross build for" ) arch = flag . String ( "arch" , "" , "Architecture to cross build for" )
cc = flag . String ( "cc" , "" , "C compiler to cross build with" ) cc = flag . String ( "cc" , "" , "C compiler to cross build with" )
) )
flag . CommandLine . Parse ( cmdline ) flag . CommandLine . Parse ( cmdline )
env := build . Env ( ) env := build . Env ( )
// Check Go version. People regularly open issues about compilation
// Check local Go version. People regularly open issues about compilation
// failure with outdated Go. This should save them the trouble.
// failure with outdated Go. This should save them the trouble.
if ! strings . Contains ( runtime . Version ( ) , "devel" ) { if ! strings . Contains ( runtime . Version ( ) , "devel" ) {
// Figure out the minor version number since we can't textually compare (1.10 < 1.9)
// Figure out the minor version number since we can't textually compare (1.10 < 1.9)
var minor int var minor int
fmt . Sscanf ( strings . TrimPrefix ( runtime . Version ( ) , "go1." ) , "%d" , & minor ) fmt . Sscanf ( strings . TrimPrefix ( runtime . Version ( ) , "go1." ) , "%d" , & minor )
if minor < 13 { if minor < 13 {
log . Println ( "You have Go version" , runtime . Version ( ) ) log . Println ( "You have Go version" , runtime . Version ( ) )
log . Println ( "go-ethereum requires at least Go version 1.13 and cannot" ) log . Println ( "go-ethereum requires at least Go version 1.13 and cannot" )
@ -218,90 +222,108 @@ func doInstall(cmdline []string) {
os . Exit ( 1 ) os . Exit ( 1 )
} }
} }
// Compile packages given as arguments, or everything if there are no arguments.
packages := [ ] string { "./..." } // Choose which go command we're going to use.
if flag . NArg ( ) > 0 { var gobuild * exec . Cmd
packages = flag . Args ( ) if ! * dlgo {
// Default behavior: use the go version which runs ci.go right now.
gobuild = goTool ( "build" )
} else {
// Download of Go requested. This is for build environments where the
// installed version is too old and cannot be upgraded easily.
cachedir := filepath . Join ( "build" , "cache" )
goroot := downloadGo ( runtime . GOARCH , runtime . GOOS , cachedir )
gobuild = localGoTool ( goroot , "build" )
} }
if * arch == "" || * arch == runtime . GOARCH { // Configure environment for cross build.
goinstall := goTool ( "install" , buildFlags ( env ) ... ) if * arch != "" || * arch != runtime . GOARCH {
if runtime . GOARCH == "arm64" { gobuild . Env = append ( gobuild . Env , "CGO_ENABLED=1" )
goinstall . Args = append ( goinstall . Args , "-p" , "1" ) gobuild . Env = append ( gobuild . Env , "GOARCH=" + * arch )
}
goinstall . Args = append ( goinstall . Args , "-trimpath" )
goinstall . Args = append ( goinstall . Args , "-v" )
goinstall . Args = append ( goinstall . Args , packages ... )
build . MustRun ( goinstall )
return
} }
// Seems we are cross compiling, work around forbidden GOBIN
// Configure C compiler.
goinstall := goToolArch ( * arch , * cc , "install" , buildFlags ( env ) ... ) if * cc == "" {
goinstall . Args = append ( goinstall . Args , "-trimpath" ) gobuild . Env = append ( gobuild . Env , "CC=" + * cc )
goinstall . Args = append ( goinstall . Args , "-v" ) } else if os . Getenv ( "CC" ) != "" {
goinstall . Args = append ( goinstall . Args , [ ] string { "-buildmode" , "archive" } ... ) gobuild . Env = append ( gobuild . Env , "CC=" + os . Getenv ( "CC" ) )
goinstall . Args = append ( goinstall . Args , packages ... ) }
build . MustRun ( goinstall )
if cmds , err := ioutil . ReadDir ( "cmd" ) ; err == nil { // arm64 CI builders are memory-constrained and can't handle concurrent builds,
for _ , cmd := range cmds { // better disable it. This check isn't the best, it should probably
pkgs , err := parser . ParseDir ( token . NewFileSet ( ) , filepath . Join ( "." , "cmd" , cmd . Name ( ) ) , nil , parser . PackageClauseOnly ) // check for something in env instead.
if err != nil { if runtime . GOARCH == "arm64" {
log . Fatal ( err ) gobuild . Args = append ( gobuild . Args , "-p" , "1" )
} }
for name := range pkgs {
if name == "main" { // Put the default settings in.
gobuild := goToolArch ( * arch , * cc , "build" , buildFlags ( env ) ... ) gobuild . Args = append ( gobuild . Args , buildFlags ( env ) ... )
gobuild . Args = append ( gobuild . Args , "-v" )
gobuild . Args = append ( gobuild . Args , [ ] string { "-o" , executablePath ( cmd . Name ( ) ) } ... ) // Show packages during build.
gobuild . Args = append ( gobuild . Args , "." + string ( filepath . Separator ) + filepath . Join ( "cmd" , cmd . Name ( ) ) ) gobuild . Args = append ( gobuild . Args , "-v" )
build . MustRun ( gobuild )
break // Now we choose what we're even building.
} // Default: collect all 'main' packages in cmd/ and build those.
} packages := flag . Args ( )
} if len ( packages ) == 0 {
packages = build . FindMainPackages ( "./cmd" )
}
// Do the build!
for _ , pkg := range packages {
args := make ( [ ] string , len ( gobuild . Args ) )
copy ( args , gobuild . Args )
args = append ( args , "-o" , executablePath ( path . Base ( pkg ) ) )
args = append ( args , pkg )
build . MustRun ( & exec . Cmd { Path : gobuild . Path , Args : args , Env : gobuild . Env } )
} }
} }
// buildFlags returns the go tool flags for building.
func buildFlags ( env build . Environment ) ( flags [ ] string ) { func buildFlags ( env build . Environment ) ( flags [ ] string ) {
var ld [ ] string var ld [ ] string
if env . Commit != "" { if env . Commit != "" {
ld = append ( ld , "-X" , "main.gitCommit=" + env . Commit ) ld = append ( ld , "-X" , "main.gitCommit=" + env . Commit )
ld = append ( ld , "-X" , "main.gitDate=" + env . Date ) ld = append ( ld , "-X" , "main.gitDate=" + env . Date )
} }
// Strip DWARF on darwin. This used to be required for certain things,
// and there is no downside to this, so we just keep doing it.
if runtime . GOOS == "darwin" { if runtime . GOOS == "darwin" {
ld = append ( ld , "-s" ) ld = append ( ld , "-s" )
} }
if len ( ld ) > 0 { if len ( ld ) > 0 {
flags = append ( flags , "-ldflags" , strings . Join ( ld , " " ) ) flags = append ( flags , "-ldflags" , strings . Join ( ld , " " ) )
} }
// We use -trimpath to avoid leaking local paths into the built executables.
flags = append ( flags , "-trimpath" )
return flags return flags
} }
// goTool returns the go tool. This uses the Go version which runs ci.go.
func goTool ( subcmd string , args ... string ) * exec . Cmd { func goTool ( subcmd string , args ... string ) * exec . Cmd {
return goToolArch ( runtime . GOARCH , os . Getenv ( "CC" ) , subcmd , args ... ) cmd := build . GoTool ( subcmd , args ... )
goToolSetEnv ( cmd )
return cmd
} }
func goToolArch ( arch string , cc string , subcmd string , args ... string ) * exec . Cmd { // localGoTool returns the go tool from the given GOROOT.
cmd := build . GoTool ( subcmd , args ... ) func localGoTool ( goroot string , subcmd string , args ... string ) * exec . Cmd {
if arch == "" || arch == runtime . GOARCH { gotool := filepath . Join ( goroot , "bin" , "go" )
cmd . Env = append ( cmd . Env , "GOBIN=" + GOBIN ) cmd := exec . Command ( gotool , subcmd )
} else { goToolSetEnv ( cmd )
cmd . Env = append ( cmd . Env , "CGO_ENABLED=1" ) cmd . Env = append ( cmd . Env , "GOROOT=" + goroot )
cmd . Env = append ( cmd . Env , "GOARCH=" + arch ) cmd . Args = append ( cmd . Args , args ... )
} return cmd
if cc != "" { }
cmd . Env = append ( cmd . Env , "CC=" + cc )
} // goToolSetEnv forwards the build environment to the go tool.
func goToolSetEnv ( cmd * exec . Cmd ) {
for _ , e := range os . Environ ( ) { for _ , e := range os . Environ ( ) {
if strings . HasPrefix ( e , "GOBIN=" ) { if strings . HasPrefix ( e , "GOBIN=" ) || strings . HasPrefix ( e , "CC=" ) {
continue continue
} }
cmd . Env = append ( cmd . Env , e ) cmd . Env = append ( cmd . Env , e )
} }
return cmd
} }
// Running The Tests
// Running The Tests
@ -363,7 +385,7 @@ func downloadLinter(cachedir string) string {
if err := csdb . DownloadFile ( url , archivePath ) ; err != nil { if err := csdb . DownloadFile ( url , archivePath ) ; err != nil {
log . Fatal ( err ) log . Fatal ( err )
} }
if err := build . ExtractTarball Archive ( archivePath , cachedir ) ; err != nil { if err := build . ExtractArchive ( archivePath , cachedir ) ; err != nil {
log . Fatal ( err ) log . Fatal ( err )
} }
return filepath . Join ( cachedir , base , "golangci-lint" ) return filepath . Join ( cachedir , base , "golangci-lint" )
@ -469,13 +491,12 @@ func maybeSkipArchive(env build.Environment) {
// Debian Packaging
// Debian Packaging
func doDebianSource ( cmdline [ ] string ) { func doDebianSource ( cmdline [ ] string ) {
var ( var (
goversion = flag . String ( "goversion" , "" , ` Go version to build with (will be included in the source package) ` ) cachedir = flag . String ( "cachedir" , "./build/cache" , ` Filesystem path to cache the downloaded Go bundles at ` )
cachedir = flag . String ( "cachedir" , "./build/cache" , ` Filesystem path to cache the downloaded Go bundles at ` ) signer = flag . String ( "signer" , "" , ` Signing key name, also used as package author ` )
signer = flag . String ( "signer" , "" , ` Signing key name, also used as package author ` ) upload = flag . String ( "upload" , "" , ` Where to upload the source package (usually "ethereum/ethereum") ` )
upload = flag . String ( "upload" , "" , ` Where to upload the source package (usually "ethereum/ethereum") ` ) sshUser = flag . String ( "sftp-user" , "" , ` Username for SFTP upload (usually "geth-ci") ` )
sshUser = flag . String ( "sftp-user" , "" , ` Username for SFTP upload (usually "geth-ci") ` ) workdir = flag . String ( "workdir" , "" , ` Output directory for packages (uses temp dir if unset) ` )
workdir = flag . String ( "workdir" , "" , ` Output directory for packages (uses temp dir if unset) ` ) now = time . Now ( )
now = time . Now ( )
) )
flag . CommandLine . Parse ( cmdline ) flag . CommandLine . Parse ( cmdline )
* workdir = makeWorkdir ( * workdir ) * workdir = makeWorkdir ( * workdir )
@ -490,7 +511,7 @@ func doDebianSource(cmdline []string) {
} }
// Download and verify the Go source package.
// Download and verify the Go source package.
gobundle := downloadGoSources ( * goversion , * cachedir ) gobundle := downloadGoSources ( * cachedir )
// Download all the dependencies needed to build the sources and run the ci script
// Download all the dependencies needed to build the sources and run the ci script
srcdepfetch := goTool ( "install" , "-n" , "./..." ) srcdepfetch := goTool ( "install" , "-n" , "./..." )
@ -509,7 +530,7 @@ func doDebianSource(cmdline []string) {
pkgdir := stageDebianSource ( * workdir , meta ) pkgdir := stageDebianSource ( * workdir , meta )
// Add Go source code
// Add Go source code
if err := build . ExtractTarball Archive ( gobundle , pkgdir ) ; err != nil { if err := build . ExtractArchive ( gobundle , pkgdir ) ; err != nil {
log . Fatalf ( "Failed to extract Go sources: %v" , err ) log . Fatalf ( "Failed to extract Go sources: %v" , err )
} }
if err := os . Rename ( filepath . Join ( pkgdir , "go" ) , filepath . Join ( pkgdir , ".go" ) ) ; err != nil { if err := os . Rename ( filepath . Join ( pkgdir , "go" ) , filepath . Join ( pkgdir , ".go" ) ) ; err != nil {
@ -541,9 +562,10 @@ func doDebianSource(cmdline []string) {
} }
} }
func downloadGoSources ( version string , cachedir string ) string { // downloadGoSources downloads the Go source tarball.
func downloadGoSources ( cachedir string ) string {
csdb := build . MustLoadChecksums ( "build/checksums.txt" ) csdb := build . MustLoadChecksums ( "build/checksums.txt" )
file := fmt . Sprintf ( "go%s.src.tar.gz" , v ersion) file := fmt . Sprintf ( "go%s.src.tar.gz" , dlgoV ersion)
url := "https://dl.google.com/go/" + file url := "https://dl.google.com/go/" + file
dst := filepath . Join ( cachedir , file ) dst := filepath . Join ( cachedir , file )
if err := csdb . DownloadFile ( url , dst ) ; err != nil { if err := csdb . DownloadFile ( url , dst ) ; err != nil {
@ -552,6 +574,41 @@ func downloadGoSources(version string, cachedir string) string {
return dst return dst
} }
// downloadGo downloads the Go binary distribution and unpacks it into a temporary
// directory. It returns the GOROOT of the unpacked toolchain.
func downloadGo ( goarch , goos , cachedir string ) string {
if goarch == "arm" {
goarch = "armv6l"
}
csdb := build . MustLoadChecksums ( "build/checksums.txt" )
file := fmt . Sprintf ( "go%s.%s-%s" , dlgoVersion , goos , goarch )
if goos == "windows" {
file += ".zip"
} else {
file += ".tar.gz"
}
url := "https://golang.org/dl/" + file
dst := filepath . Join ( cachedir , file )
if err := csdb . DownloadFile ( url , dst ) ; err != nil {
log . Fatal ( err )
}
ucache , err := os . UserCacheDir ( )
if err != nil {
log . Fatal ( err )
}
godir := filepath . Join ( ucache , fmt . Sprintf ( "geth-go-%s-%s-%s" , dlgoVersion , goos , goarch ) )
if err := build . ExtractArchive ( dst , godir ) ; err != nil {
log . Fatal ( err )
}
goroot , err := filepath . Abs ( filepath . Join ( godir , "go" ) )
if err != nil {
log . Fatal ( err )
}
return goroot
}
func ppaUpload ( workdir , ppa , sshUser string , files [ ] string ) { func ppaUpload ( workdir , ppa , sshUser string , files [ ] string ) {
p := strings . Split ( ppa , "/" ) p := strings . Split ( ppa , "/" )
if len ( p ) != 2 { if len ( p ) != 2 {
Felix Lange
4 years ago
committed by
GitHub