diff --git a/go/loader/cgo.go b/go/loader/cgo.go new file mode 100644 index 00000000..b30d4adb --- /dev/null +++ b/go/loader/cgo.go @@ -0,0 +1,194 @@ +package loader + +// This file handles cgo preprocessing of files containing `import "C"`. +// +// DESIGN +// +// The approach taken is to run the cgo processor on the package's +// CgoFiles and parse the output, faking the filenames of the +// resulting ASTs so that the synthetic file containing the C types is +// called "C" (e.g. "~/go/src/pkg/net/C") and the preprocessed files +// have their original names (e.g. "~/go/src/pkg/net/cgo_unix.go"), +// not the names of the actual temporary files. +// +// The advantage of this approach is its fidelity to 'go build'. The +// downside is that the token.Position.Offset for each AST node is +// incorrect, being an offset within the temporary file. Line numbers +// should still be correct because of the //line comments. +// +// The logic of this file is mostly plundered from the 'go build' +// tool, which also invokes the cgo preprocessor. +// +// +// REJECTED ALTERNATIVE +// +// An alternative approach that we explored is to extend go/types' +// Importer mechanism to provide the identity of the importing package +// so that each time `import "C"` appears it resolves to a different +// synthetic package containing just the objects needed in that case. +// The loader would invoke cgo but parse only the cgo_types.go file +// defining the package-level objects, discarding the other files +// resulting from preprocessing. +// +// The benefit of this approach would have been that source-level +// syntax information would correspond exactly to the original cgo +// file, with no preprocessing involved, making source tools like +// godoc, oracle, and eg happy. However, the approach was rejected +// due to the additional complexity it would impose on go/types. (It +// made for a beautiful demo, though.) +// +// cgo files, despite their *.go extension, are not legal Go source +// files per the specification since they may refer to unexported +// members of package "C" such as C.int. Also, a function such as +// C.getpwent has in effect two types, one matching its C type and one +// which additionally returns (errno C.int). The cgo preprocessor +// uses name mangling to distinguish these two functions in the +// processed code, but go/types would need to duplicate this logic in +// its handling of function calls, analogous to the treatment of map +// lookups in which y=m[k] and y,ok=m[k] are both legal. + +import ( + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" +) + +// processCgoFiles invokes the cgo preprocessor on bp.CgoFiles, parses +// the output and returns the resulting ASTs. +// +func processCgoFiles(bp *build.Package, fset *token.FileSet, mode parser.Mode) ([]*ast.File, error) { + tmpdir, err := ioutil.TempDir("", strings.Replace(bp.ImportPath, "/", "_", -1)+"_C") + if err != nil { + return nil, err + } + defer os.RemoveAll(tmpdir) + + cgoFiles, cgoDisplayFiles, err := runCgo(bp, tmpdir) + if err != nil { + return nil, err + } + var files []*ast.File + for i := range cgoFiles { + rd, err := os.Open(cgoFiles[i]) + if err != nil { + return nil, err + } + defer rd.Close() + display := filepath.Join(bp.Dir, cgoDisplayFiles[i]) + f, err := parser.ParseFile(fset, display, rd, mode) + if err != nil { + return nil, err + } + files = append(files, f) + } + return files, nil +} + +var cgoRe = regexp.MustCompile(`[/\\:]`) + +// runCgo invokes the cgo preprocessor on bp.CgoFiles and returns two +// lists of files: the resulting processed files (in temporary +// directory tmpdir) and the corresponding names of the unprocessed files. +// +// runCgo is adapted from (*builder).cgo in +// $GOROOT/src/cmd/go/build.go, but these features are unsupported: +// pkg-config, Objective C, CGOPKGPATH, CGO_FLAGS. +// +func runCgo(bp *build.Package, tmpdir string) (files, displayFiles []string, err error) { + cgoCPPFLAGS, _, _, _ := cflags(bp, true) + _, cgoexeCFLAGS, _, _ := cflags(bp, false) + + if len(bp.CgoPkgConfig) > 0 { + return nil, nil, fmt.Errorf("cgo pkg-config not supported") + } + + // Allows including _cgo_export.h from .[ch] files in the package. + cgoCPPFLAGS = append(cgoCPPFLAGS, "-I", tmpdir) + + // _cgo_gotypes.go (displayed "C") contains the type definitions. + files = append(files, filepath.Join(tmpdir, "_cgo_gotypes.go")) + displayFiles = append(displayFiles, "C") + for _, fn := range bp.CgoFiles { + // "foo.cgo1.go" (displayed "foo.go") is the processed Go source. + f := cgoRe.ReplaceAllString(fn[:len(fn)-len("go")], "_") + files = append(files, filepath.Join(tmpdir, f+"cgo1.go")) + displayFiles = append(displayFiles, fn) + } + + var cgoflags []string + if bp.Goroot && bp.ImportPath == "runtime/cgo" { + cgoflags = append(cgoflags, "-import_runtime_cgo=false") + } + if bp.Goroot && bp.ImportPath == "runtime/race" || bp.ImportPath == "runtime/cgo" { + cgoflags = append(cgoflags, "-import_syscall=false") + } + + args := stringList( + "go", "tool", "cgo", "-objdir", tmpdir, cgoflags, "--", + cgoCPPFLAGS, cgoexeCFLAGS, bp.CgoFiles, + ) + if false { + log.Printf("Running cgo for package %q: %s", bp.ImportPath, args) + } + cmd := exec.Command(args[0], args[1:]...) + cmd.Dir = bp.Dir + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return nil, nil, fmt.Errorf("cgo failed: %s: %s", args, err) + } + + return files, displayFiles, nil +} + +// -- unmodified from 'go build' --------------------------------------- + +// Return the flags to use when invoking the C or C++ compilers, or cgo. +func cflags(p *build.Package, def bool) (cppflags, cflags, cxxflags, ldflags []string) { + var defaults string + if def { + defaults = "-g -O2" + } + + cppflags = stringList(envList("CGO_CPPFLAGS", ""), p.CgoCPPFLAGS) + cflags = stringList(envList("CGO_CFLAGS", defaults), p.CgoCFLAGS) + cxxflags = stringList(envList("CGO_CXXFLAGS", defaults), p.CgoCXXFLAGS) + ldflags = stringList(envList("CGO_LDFLAGS", defaults), p.CgoLDFLAGS) + return +} + +// envList returns the value of the given environment variable broken +// into fields, using the default value when the variable is empty. +func envList(key, def string) []string { + v := os.Getenv(key) + if v == "" { + v = def + } + return strings.Fields(v) +} + +// stringList's arguments should be a sequence of string or []string values. +// stringList flattens them into a single []string. +func stringList(args ...interface{}) []string { + var x []string + for _, arg := range args { + switch arg := arg.(type) { + case []string: + x = append(x, arg...) + case string: + x = append(x, arg) + default: + panic("stringList: invalid argument") + } + } + return x +} diff --git a/go/loader/loader.go b/go/loader/loader.go index 04b54cb3..534cfd1f 100644 --- a/go/loader/loader.go +++ b/go/loader/loader.go @@ -186,6 +186,11 @@ type Config struct { // If Build is non-nil, it is used to locate source packages. // Otherwise &build.Default is used. + // + // By default, cgo is invoked to preprocess Go files that + // import the fake package "C". This behaviour can be + // disabled by setting CGO_ENABLED=0 in the environment prior + // to startup, or by setting Build.CgoEnabled=false. Build *build.Context // If DisplayPath is non-nil, it is used to transform each @@ -612,18 +617,15 @@ func (conf *Config) build() *build.Context { // then loads, parses and returns them. // // 'which' indicates which files to include: -// 'g': include non-test *.go source files (GoFiles) +// 'g': include non-test *.go source files (GoFiles + processed CgoFiles) // 't': include in-package *_test.go source files (TestGoFiles) // 'x': include external *_test.go source files. (XTestGoFiles) // +// TODO(adonovan): don't fail just because some files contain parse errors. +// func (conf *Config) parsePackageFiles(path string, which rune) ([]*ast.File, error) { - // Set the "!cgo" go/build tag, preferring (dummy) Go to - // native C implementations of net.cgoLookupHost et al. - ctxt := *conf.build() // copy - ctxt.CgoEnabled = false - // Import(srcDir="") disables local imports, e.g. import "./foo". - bp, err := ctxt.Import(path, "", 0) + bp, err := conf.build().Import(path, "", 0) if _, ok := err.(*build.NoGoError); ok { return nil, nil // empty directory } @@ -642,7 +644,22 @@ func (conf *Config) parsePackageFiles(path string, which rune) ([]*ast.File, err default: panic(which) } - return parseFiles(conf.fset(), &ctxt, conf.DisplayPath, bp.Dir, filenames, conf.ParserMode) + + files, err := parseFiles(conf.fset(), conf.build(), conf.DisplayPath, bp.Dir, filenames, conf.ParserMode) + if err != nil { + return nil, err + } + + // Preprocess CgoFiles and parse the outputs (sequentially). + if which == 'g' && bp.CgoFiles != nil { + cgofiles, err := processCgoFiles(bp, conf.fset(), conf.ParserMode) + if err != nil { + return nil, err + } + files = append(files, cgofiles...) + } + + return files, nil } // doImport imports the package denoted by path. diff --git a/go/loader/stdlib_test.go b/go/loader/stdlib_test.go index e8d93637..7a484b1f 100644 --- a/go/loader/stdlib_test.go +++ b/go/loader/stdlib_test.go @@ -11,9 +11,12 @@ package loader_test // Run test with GOMAXPROCS=8. import ( + "bytes" "fmt" "go/ast" + "go/build" "go/token" + "io/ioutil" "os" "path/filepath" "runtime" @@ -133,3 +136,70 @@ func TestStdlib(t *testing.T) { t.Log("Load/parse/typecheck: ", t1.Sub(t0)) t.Log("#MB: ", int64(memstats.Alloc-alloc)/1000000) } + +func TestCgoOption(t *testing.T) { + // Test that we can load cgo-using packages with + // CGO_ENABLED=[01], which causes go/build to select pure + // Go/native implementations, respectively, based on build + // tags. + // + // Each entry specifies a package-level object and the generic + // file expected to define it when cgo is disabled. + // When cgo is enabled, the exact file is not specified (since + // it varies by platform), but must differ from the generic one. + // + // The test also loads the actual file to verify that the + // object is indeed defined at that location. + for _, test := range []struct { + pkg, name, genericFile string + }{ + {"net", "cgoLookupHost", "cgo_stub.go"}, + {"os/user", "lookupId", "lookup_stubs.go"}, + } { + ctxt := build.Default + for _, ctxt.CgoEnabled = range []bool{false, true} { + conf := loader.Config{Build: &ctxt} + conf.Import(test.pkg) + prog, err := conf.Load() + if err != nil { + t.Errorf("Load failed: %v", err) + continue + } + info := prog.Imported[test.pkg] + if info == nil { + t.Errorf("package %s not found", test.pkg) + continue + } + obj := info.Pkg.Scope().Lookup(test.name) + if obj == nil { + t.Errorf("no object %s.%s", test.pkg, test.name) + continue + } + posn := prog.Fset.Position(obj.Pos()) + t.Logf("%s: %s (CgoEnabled=%t)", posn, obj, ctxt.CgoEnabled) + + gotFile := filepath.Base(posn.Filename) + filesMatch := gotFile == test.genericFile + + if ctxt.CgoEnabled && filesMatch { + t.Errorf("CGO_ENABLED=1: %s found in %s, want native file", + obj, gotFile) + } else if !ctxt.CgoEnabled && !filesMatch { + t.Errorf("CGO_ENABLED=0: %s found in %s, want %s", + obj, gotFile, test.genericFile) + } + + // Load the file and check the object is declared at the right place. + b, err := ioutil.ReadFile(posn.Filename) + if err != nil { + t.Errorf("can't read %s: %s", posn.Filename, err) + continue + } + line := string(bytes.Split(b, []byte("\n"))[posn.Line-1]) + ident := line[posn.Column-1:] + if !strings.HasPrefix(ident, test.name) { + t.Errorf("%s: %s not declared here (looking at %q)", posn, obj, ident) + } + } + } +}