Source file src/cmd/go/internal/work/buildid.go
1 // Copyright 2017 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package work 6 7 import ( 8 "bytes" 9 "fmt" 10 "os" 11 "os/exec" 12 "strings" 13 "sync" 14 15 "cmd/go/internal/base" 16 "cmd/go/internal/cache" 17 "cmd/go/internal/cfg" 18 "cmd/go/internal/fsys" 19 "cmd/go/internal/str" 20 "cmd/internal/buildid" 21 "cmd/internal/pathcache" 22 "cmd/internal/quoted" 23 "cmd/internal/telemetry/counter" 24 ) 25 26 // Build IDs 27 // 28 // Go packages and binaries are stamped with build IDs that record both 29 // the action ID, which is a hash of the inputs to the action that produced 30 // the packages or binary, and the content ID, which is a hash of the action 31 // output, namely the archive or binary itself. The hash is the same one 32 // used by the build artifact cache (see cmd/go/internal/cache), but 33 // truncated when stored in packages and binaries, as the full length is not 34 // needed and is a bit unwieldy. The precise form is 35 // 36 // actionID/[.../]contentID 37 // 38 // where the actionID and contentID are prepared by buildid.HashToString below. 39 // and are found by looking for the first or last slash. 40 // Usually the buildID is simply actionID/contentID, but see below for an 41 // exception. 42 // 43 // The build ID serves two primary purposes. 44 // 45 // 1. The action ID half allows installed packages and binaries to serve as 46 // one-element cache entries. If we intend to build math.a with a given 47 // set of inputs summarized in the action ID, and the installed math.a already 48 // has that action ID, we can reuse the installed math.a instead of rebuilding it. 49 // 50 // 2. The content ID half allows the easy preparation of action IDs for steps 51 // that consume a particular package or binary. The content hash of every 52 // input file for a given action must be included in the action ID hash. 53 // Storing the content ID in the build ID lets us read it from the file with 54 // minimal I/O, instead of reading and hashing the entire file. 55 // This is especially effective since packages and binaries are typically 56 // the largest inputs to an action. 57 // 58 // Separating action ID from content ID is important for reproducible builds. 59 // The compiler is compiled with itself. If an output were represented by its 60 // own action ID (instead of content ID) when computing the action ID of 61 // the next step in the build process, then the compiler could never have its 62 // own input action ID as its output action ID (short of a miraculous hash collision). 63 // Instead we use the content IDs to compute the next action ID, and because 64 // the content IDs converge, so too do the action IDs and therefore the 65 // build IDs and the overall compiler binary. See cmd/dist's cmdbootstrap 66 // for the actual convergence sequence. 67 // 68 // The “one-element cache” purpose is a bit more complex for installed 69 // binaries. For a binary, like cmd/gofmt, there are two steps: compile 70 // cmd/gofmt/*.go into main.a, and then link main.a into the gofmt binary. 71 // We do not install gofmt's main.a, only the gofmt binary. Being able to 72 // decide that the gofmt binary is up-to-date means computing the action ID 73 // for the final link of the gofmt binary and comparing it against the 74 // already-installed gofmt binary. But computing the action ID for the link 75 // means knowing the content ID of main.a, which we did not keep. 76 // To sidestep this problem, each binary actually stores an expanded build ID: 77 // 78 // actionID(binary)/actionID(main.a)/contentID(main.a)/contentID(binary) 79 // 80 // (Note that this can be viewed equivalently as: 81 // 82 // actionID(binary)/buildID(main.a)/contentID(binary) 83 // 84 // Storing the buildID(main.a) in the middle lets the computations that care 85 // about the prefix or suffix halves ignore the middle and preserves the 86 // original build ID as a contiguous string.) 87 // 88 // During the build, when it's time to build main.a, the gofmt binary has the 89 // information needed to decide whether the eventual link would produce 90 // the same binary: if the action ID for main.a's inputs matches and then 91 // the action ID for the link step matches when assuming the given main.a 92 // content ID, then the binary as a whole is up-to-date and need not be rebuilt. 93 // 94 // This is all a bit complex and may be simplified once we can rely on the 95 // main cache, but at least at the start we will be using the content-based 96 // staleness determination without a cache beyond the usual installed 97 // package and binary locations. 98 99 const buildIDSeparator = "/" 100 101 // actionID returns the action ID half of a build ID. 102 func actionID(buildID string) string { 103 i := strings.Index(buildID, buildIDSeparator) 104 if i < 0 { 105 return buildID 106 } 107 return buildID[:i] 108 } 109 110 // contentID returns the content ID half of a build ID. 111 func contentID(buildID string) string { 112 return buildID[strings.LastIndex(buildID, buildIDSeparator)+1:] 113 } 114 115 // toolID returns the unique ID to use for the current copy of the 116 // named tool (asm, compile, cover, link). 117 // 118 // It is important that if the tool changes (for example a compiler bug is fixed 119 // and the compiler reinstalled), toolID returns a different string, so that old 120 // package archives look stale and are rebuilt (with the fixed compiler). 121 // This suggests using a content hash of the tool binary, as stored in the build ID. 122 // 123 // Unfortunately, we can't just open the tool binary, because the tool might be 124 // invoked via a wrapper program specified by -toolexec and we don't know 125 // what the wrapper program does. In particular, we want "-toolexec toolstash" 126 // to continue working: it does no good if "-toolexec toolstash" is executing a 127 // stashed copy of the compiler but the go command is acting as if it will run 128 // the standard copy of the compiler. The solution is to ask the tool binary to tell 129 // us its own build ID using the "-V=full" flag now supported by all tools. 130 // Then we know we're getting the build ID of the compiler that will actually run 131 // during the build. (How does the compiler binary know its own content hash? 132 // We store it there using updateBuildID after the standard link step.) 133 // 134 // A final twist is that we'd prefer to have reproducible builds for release toolchains. 135 // It should be possible to cross-compile for Windows from either Linux or Mac 136 // or Windows itself and produce the same binaries, bit for bit. If the tool ID, 137 // which influences the action ID half of the build ID, is based on the content ID, 138 // then the Linux compiler binary and Mac compiler binary will have different tool IDs 139 // and therefore produce executables with different action IDs. 140 // To avoid this problem, for releases we use the release version string instead 141 // of the compiler binary's content hash. This assumes that all compilers built 142 // on all different systems are semantically equivalent, which is of course only true 143 // modulo bugs. (Producing the exact same executables also requires that the different 144 // build setups agree on details like $GOROOT and file name paths, but at least the 145 // tool IDs do not make it impossible.) 146 func (b *Builder) toolID(name string) string { 147 return b.toolIDCache.Do(name, func() string { 148 path := base.Tool(name) 149 desc := "go tool " + name 150 151 // Special case: -{vet,fix}tool overrides usual cmd/{vet,fix} 152 // for testing or supplying an alternative analysis tool. 153 // (We use only "vet" terminology in the action graph.) 154 if name == "vet" { 155 path = VetTool 156 desc = VetTool 157 } 158 159 cmdline := str.StringList(cfg.BuildToolexec, path, "-V=full") 160 cmd := exec.Command(cmdline[0], cmdline[1:]...) 161 var stdout, stderr strings.Builder 162 cmd.Stdout = &stdout 163 cmd.Stderr = &stderr 164 if err := cmd.Run(); err != nil { 165 if stderr.Len() > 0 { 166 os.Stderr.WriteString(stderr.String()) 167 } 168 base.Fatalf("go: error obtaining buildID for %s: %v", desc, err) 169 } 170 171 line := stdout.String() 172 f := strings.Fields(line) 173 if len(f) < 3 || f[0] != name && path != VetTool || f[1] != "version" || strings.Contains(f[2], "devel") && !strings.HasPrefix(f[len(f)-1], "buildID=") { 174 base.Fatalf("go: parsing buildID from %s -V=full: unexpected output:\n\t%s", desc, line) 175 } 176 if strings.Contains(f[2], "devel") { 177 // On the development branch, use the content ID part of the build ID. 178 return contentID(f[len(f)-1]) 179 } 180 // For a release, the output is like: "compile version go1.9.1 X:framepointer". 181 // Use the whole line. 182 return strings.TrimSpace(line) 183 }) 184 } 185 186 // gccToolID returns the unique ID to use for a tool that is invoked 187 // by the GCC driver. This is used particularly for gccgo, but this can also 188 // be used for gcc, g++, gfortran, etc.; those tools all use the GCC 189 // driver under different names. The approach used here should also 190 // work for sufficiently new versions of clang. Unlike toolID, the 191 // name argument is the program to run. The language argument is the 192 // type of input file as passed to the GCC driver's -x option. 193 // 194 // For these tools we have no -V=full option to dump the build ID, 195 // but we can run the tool with -v -### to reliably get the compiler proper 196 // and hash that. That will work in the presence of -toolexec. 197 // 198 // In order to get reproducible builds for released compilers, we 199 // detect a released compiler by the absence of "experimental" in the 200 // --version output, and in that case we just use the version string. 201 // 202 // gccToolID also returns the underlying executable for the compiler. 203 // The caller assumes that stat of the exe can be used, combined with the id, 204 // to detect changes in the underlying compiler. The returned exe can be empty, 205 // which means to rely only on the id. 206 func (b *Builder) gccToolID(name, language string) (id, exe string, err error) { 207 //TODO: Use par.Cache instead of a mutex and a map. See Builder.toolID. 208 key := name + "." + language 209 b.id.Lock() 210 id = b.gccToolIDCache[key] 211 exe = b.gccToolIDCache[key+".exe"] 212 b.id.Unlock() 213 214 if id != "" { 215 return id, exe, nil 216 } 217 218 // Invoke the driver with -### to see the subcommands and the 219 // version strings. Use -x to set the language. Pretend to 220 // compile an empty file on standard input. 221 cmdline := str.StringList(cfg.BuildToolexec, name, "-###", "-x", language, "-c", "-") 222 cmd := exec.Command(cmdline[0], cmdline[1:]...) 223 // Force untranslated output so that we see the string "version". 224 cmd.Env = append(os.Environ(), "LC_ALL=C") 225 out, err := cmd.CombinedOutput() 226 if err != nil { 227 return "", "", fmt.Errorf("%s: %v; output: %q", name, err, out) 228 } 229 230 version := "" 231 lines := strings.Split(string(out), "\n") 232 for _, line := range lines { 233 fields := strings.Fields(line) 234 for i, field := range fields { 235 if strings.HasSuffix(field, ":") { 236 // Avoid parsing fields of lines like "Configured with: …", which may 237 // contain arbitrary substrings. 238 break 239 } 240 if field == "version" && i < len(fields)-1 { 241 // Check that the next field is plausibly a version number. 242 // We require only that it begins with an ASCII digit, 243 // since we don't know what version numbering schemes a given 244 // C compiler may use. (Clang and GCC mostly seem to follow the scheme X.Y.Z, 245 // but in https://go.dev/issue/64619 we saw "8.3 [DragonFly]", and who knows 246 // what other C compilers like "zig cc" might report?) 247 next := fields[i+1] 248 if len(next) > 0 && next[0] >= '0' && next[0] <= '9' { 249 version = line 250 break 251 } 252 } 253 } 254 if version != "" { 255 break 256 } 257 } 258 if version == "" { 259 return "", "", fmt.Errorf("%s: can not find version number in %q", name, out) 260 } 261 262 if !strings.Contains(version, "experimental") { 263 // This is a release. Use this line as the tool ID. 264 id = version 265 } else { 266 // This is a development version. The first line with 267 // a leading space is the compiler proper. 268 compiler := "" 269 for _, line := range lines { 270 if strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " (in-process)") { 271 compiler = line 272 break 273 } 274 } 275 if compiler == "" { 276 return "", "", fmt.Errorf("%s: can not find compilation command in %q", name, out) 277 } 278 279 fields, _ := quoted.Split(compiler) 280 if len(fields) == 0 { 281 return "", "", fmt.Errorf("%s: compilation command confusion %q", name, out) 282 } 283 exe = fields[0] 284 if !strings.ContainsAny(exe, `/\`) { 285 if lp, err := pathcache.LookPath(exe); err == nil { 286 exe = lp 287 } 288 } 289 id, err = buildid.ReadFile(exe) 290 if err != nil { 291 return "", "", err 292 } 293 294 // If we can't find a build ID, use a hash. 295 if id == "" { 296 id = b.fileHash(exe) 297 } 298 } 299 300 b.id.Lock() 301 b.gccToolIDCache[key] = id 302 b.gccToolIDCache[key+".exe"] = exe 303 b.id.Unlock() 304 305 return id, exe, nil 306 } 307 308 // Check if assembler used by gccgo is GNU as. 309 func assemblerIsGas() bool { 310 cmd := exec.Command(BuildToolchain.compiler(), "-print-prog-name=as") 311 assembler, err := cmd.Output() 312 if err == nil { 313 cmd := exec.Command(strings.TrimSpace(string(assembler)), "--version") 314 out, err := cmd.Output() 315 return err == nil && strings.Contains(string(out), "GNU") 316 } else { 317 return false 318 } 319 } 320 321 // gccgoBuildIDFile creates an assembler file that records the 322 // action's build ID in an SHF_EXCLUDE section for ELF files or 323 // in a CSECT in XCOFF files. 324 func (b *Builder) gccgoBuildIDFile(a *Action) (string, error) { 325 sfile := a.Objdir + "_buildid.s" 326 327 var buf bytes.Buffer 328 if cfg.Goos == "aix" { 329 fmt.Fprintf(&buf, "\t.csect .go.buildid[XO]\n") 330 } else if (cfg.Goos != "solaris" && cfg.Goos != "illumos") || assemblerIsGas() { 331 fmt.Fprintf(&buf, "\t"+`.section .go.buildid,"e"`+"\n") 332 } else if cfg.Goarch == "sparc" || cfg.Goarch == "sparc64" { 333 fmt.Fprintf(&buf, "\t"+`.section ".go.buildid",#exclude`+"\n") 334 } else { // cfg.Goarch == "386" || cfg.Goarch == "amd64" 335 fmt.Fprintf(&buf, "\t"+`.section .go.buildid,#exclude`+"\n") 336 } 337 fmt.Fprintf(&buf, "\t.byte ") 338 for i := 0; i < len(a.buildID); i++ { 339 if i > 0 { 340 if i%8 == 0 { 341 fmt.Fprintf(&buf, "\n\t.byte ") 342 } else { 343 fmt.Fprintf(&buf, ",") 344 } 345 } 346 fmt.Fprintf(&buf, "%#02x", a.buildID[i]) 347 } 348 fmt.Fprintf(&buf, "\n") 349 if cfg.Goos != "solaris" && cfg.Goos != "illumos" && cfg.Goos != "aix" { 350 secType := "@progbits" 351 if cfg.Goarch == "arm" { 352 secType = "%progbits" 353 } 354 fmt.Fprintf(&buf, "\t"+`.section .note.GNU-stack,"",%s`+"\n", secType) 355 fmt.Fprintf(&buf, "\t"+`.section .note.GNU-split-stack,"",%s`+"\n", secType) 356 } 357 358 if err := b.Shell(a).writeFile(sfile, buf.Bytes()); err != nil { 359 return "", err 360 } 361 362 return sfile, nil 363 } 364 365 // buildID returns the build ID found in the given file. 366 // If no build ID is found, buildID returns the content hash of the file. 367 func (b *Builder) buildID(file string) string { 368 b.id.Lock() 369 id := b.buildIDCache[file] 370 b.id.Unlock() 371 372 if id != "" { 373 return id 374 } 375 376 id, err := buildid.ReadFile(file) 377 if err != nil { 378 id = b.fileHash(file) 379 } 380 381 b.id.Lock() 382 b.buildIDCache[file] = id 383 b.id.Unlock() 384 385 return id 386 } 387 388 // fileHash returns the content hash of the named file. 389 func (b *Builder) fileHash(file string) string { 390 sum, err := cache.FileHash(fsys.Actual(file)) 391 if err != nil { 392 return "" 393 } 394 return buildid.HashToString(sum) 395 } 396 397 var ( 398 counterCacheHit = counter.New("go/buildcache/hit") 399 counterCacheMiss = counter.New("go/buildcache/miss") 400 401 stdlibRecompiled = counter.New("go/buildcache/stdlib-recompiled") 402 stdlibRecompiledIncOnce = sync.OnceFunc(stdlibRecompiled.Inc) 403 ) 404 405 // testRunAction returns the run action for a test given the link action 406 // for the test binary, if the only (non-test-barrier) action that depend 407 // on the link action is the run action. 408 func testRunAction(a *Action) *Action { 409 if len(a.triggers) != 1 || a.triggers[0].Mode != "test barrier" { 410 return nil 411 } 412 var runAction *Action 413 for _, t := range a.triggers[0].triggers { 414 if t.Mode == "test run" { 415 if runAction != nil { 416 return nil 417 } 418 runAction = t 419 } 420 } 421 return runAction 422 } 423 424 // useCache tries to satisfy the action a, which has action ID actionHash, 425 // by using a cached result from an earlier build. 426 // If useCache decides that the cache can be used, it sets a.buildID 427 // and a.built for use by parent actions and then returns true. 428 // Otherwise it sets a.buildID to a temporary build ID for use in the build 429 // and returns false. When useCache returns false the expectation is that 430 // the caller will build the target and then call updateBuildID to finish the 431 // build ID computation. 432 // When useCache returns false, it may have initiated buffering of output 433 // during a's work. The caller should defer b.flushOutput(a), to make sure 434 // that flushOutput is eventually called regardless of whether the action 435 // succeeds. The flushOutput call must happen after updateBuildID. 436 func (b *Builder) useCache(a *Action, actionHash cache.ActionID, target string, printOutput bool) (ok bool) { 437 // The second half of the build ID here is a placeholder for the content hash. 438 // It's important that the overall buildID be unlikely verging on impossible 439 // to appear in the output by chance, but that should be taken care of by 440 // the actionID half; if it also appeared in the input that would be like an 441 // engineered 120-bit partial SHA256 collision. 442 a.actionID = actionHash 443 actionID := buildid.HashToString(actionHash) 444 if a.json != nil { 445 a.json.ActionID = actionID 446 } 447 contentID := actionID // temporary placeholder, likely unique 448 a.buildID = actionID + buildIDSeparator + contentID 449 450 // Executable binaries also record the main build ID in the middle. 451 // See "Build IDs" comment above. 452 if a.Mode == "link" { 453 mainpkg := a.Deps[0] 454 a.buildID = actionID + buildIDSeparator + mainpkg.buildID + buildIDSeparator + contentID 455 } 456 457 // If user requested -a, we force a rebuild, so don't use the cache. 458 if cfg.BuildA { 459 if p := a.Package; p != nil && !p.Stale { 460 p.Stale = true 461 p.StaleReason = "build -a flag in use" 462 } 463 // Begin saving output for later writing to cache. 464 a.output = []byte{} 465 return false 466 } 467 468 defer func() { 469 // Increment counters for cache hits and misses based on the return value 470 // of this function. Don't increment counters if we return early because of 471 // cfg.BuildA above because we don't even look at the cache in that case. 472 if ok { 473 counterCacheHit.Inc() 474 } else { 475 if a.Package != nil && a.Package.Standard { 476 stdlibRecompiledIncOnce() 477 } 478 counterCacheMiss.Inc() 479 } 480 }() 481 482 c := cache.Default() 483 484 if target != "" { 485 buildID, _ := buildid.ReadFile(target) 486 if strings.HasPrefix(buildID, actionID+buildIDSeparator) { 487 a.buildID = buildID 488 if a.json != nil { 489 a.json.BuildID = a.buildID 490 } 491 a.built = target 492 // Poison a.Target to catch uses later in the build. 493 a.Target = "DO NOT USE - " + a.Mode 494 return true 495 } 496 // Special case for building a main package: if the only thing we 497 // want the package for is to link a binary, and the binary is 498 // already up-to-date, then to avoid a rebuild, report the package 499 // as up-to-date as well. See "Build IDs" comment above. 500 // TODO(rsc): Rewrite this code to use a TryCache func on the link action. 501 if !b.NeedExport && a.Mode == "build" && len(a.triggers) == 1 && a.triggers[0].Mode == "link" { 502 if id := strings.Split(buildID, buildIDSeparator); len(id) == 4 && id[1] == actionID { 503 // Temporarily assume a.buildID is the package build ID 504 // stored in the installed binary, and see if that makes 505 // the upcoming link action ID a match. If so, report that 506 // we built the package, safe in the knowledge that the 507 // link step will not ask us for the actual package file. 508 // Note that (*Builder).LinkAction arranged that all of 509 // a.triggers[0]'s dependencies other than a are also 510 // dependencies of a, so that we can be sure that, 511 // other than a.buildID, b.linkActionID is only accessing 512 // build IDs of completed actions. 513 oldBuildID := a.buildID 514 a.buildID = id[1] + buildIDSeparator + id[2] 515 linkID := buildid.HashToString(b.linkActionID(a.triggers[0])) 516 if id[0] == linkID { 517 // Best effort attempt to display output from the compile and link steps. 518 // If it doesn't work, it doesn't work: reusing the cached binary is more 519 // important than reprinting diagnostic information. 520 if printOutput { 521 showStdout(b, c, a, "stdout") // compile output 522 showStdout(b, c, a, "link-stdout") // link output 523 } 524 525 // Poison a.Target to catch uses later in the build. 526 a.Target = "DO NOT USE - main build pseudo-cache Target" 527 a.built = "DO NOT USE - main build pseudo-cache built" 528 if a.json != nil { 529 a.json.BuildID = a.buildID 530 } 531 return true 532 } 533 // Otherwise restore old build ID for main build. 534 a.buildID = oldBuildID 535 } 536 } 537 } 538 539 // TODO(matloob): If we end up caching all executables, the test executable will 540 // already be cached so building it won't do any work. But for now we won't 541 // cache all executables and instead only want to cache some: 542 // we only cache executables produced for 'go run' (and soon, for 'go tool'). 543 // 544 // Special case for linking a test binary: if the only thing we 545 // want the binary for is to run the test, and the test result is cached, 546 // then to avoid the link step, report the link as up-to-date. 547 // We avoid the nested build ID problem in the previous special case 548 // by recording the test results in the cache under the action ID half. 549 if ra := testRunAction(a); ra != nil && ra.TryCache != nil && ra.TryCache(b, ra, a) { 550 // Best effort attempt to display output from the compile and link steps. 551 // If it doesn't work, it doesn't work: reusing the test result is more 552 // important than reprinting diagnostic information. 553 if printOutput { 554 showStdout(b, c, a.Deps[0], "stdout") // compile output 555 showStdout(b, c, a.Deps[0], "link-stdout") // link output 556 } 557 558 // Poison a.Target to catch uses later in the build. 559 a.Target = "DO NOT USE - pseudo-cache Target" 560 a.built = "DO NOT USE - pseudo-cache built" 561 return true 562 } 563 564 // Check to see if the action output is cached. 565 if file, _, err := cache.GetFile(c, actionHash); err == nil { 566 if a.Mode == "preprocess PGO profile" { 567 // Preprocessed PGO profiles don't embed a build ID, so 568 // skip the build ID lookup. 569 // TODO(prattmic): better would be to add a build ID to the format. 570 a.built = file 571 a.Target = "DO NOT USE - using cache" 572 return true 573 } 574 if buildID, err := buildid.ReadFile(file); err == nil { 575 if printOutput { 576 switch a.Mode { 577 case "link": 578 // The link output is stored using the build action's action ID. 579 // See corresponding code storing the link output in updateBuildID. 580 for _, a1 := range a.Deps { 581 showStdout(b, c, a1, "link-stdout") // link output 582 } 583 default: 584 showStdout(b, c, a, "stdout") // compile output 585 } 586 } 587 a.built = file 588 a.Target = "DO NOT USE - using cache" 589 a.buildID = buildID 590 if a.json != nil { 591 a.json.BuildID = a.buildID 592 } 593 if p := a.Package; p != nil && target != "" { 594 p.Stale = true 595 // Clearer than explaining that something else is stale. 596 p.StaleReason = "not installed but available in build cache" 597 } 598 return true 599 } 600 } 601 602 // If we've reached this point, we can't use the cache for the action. 603 if p := a.Package; p != nil && !p.Stale { 604 p.Stale = true 605 p.StaleReason = "build ID mismatch" 606 if b.IsCmdList { 607 // Since we may end up printing StaleReason, include more detail. 608 for _, p1 := range p.Internal.Imports { 609 if p1.Stale && p1.StaleReason != "" { 610 if strings.HasPrefix(p1.StaleReason, "stale dependency: ") { 611 p.StaleReason = p1.StaleReason 612 break 613 } 614 if strings.HasPrefix(p.StaleReason, "build ID mismatch") { 615 p.StaleReason = "stale dependency: " + p1.ImportPath 616 } 617 } 618 } 619 } 620 } 621 622 // Begin saving output for later writing to cache. 623 a.output = []byte{} 624 return false 625 } 626 627 func showStdout(b *Builder, c cache.Cache, a *Action, key string) error { 628 actionID := a.actionID 629 630 stdout, stdoutEntry, err := cache.GetBytes(c, cache.Subkey(actionID, key)) 631 if err != nil { 632 return err 633 } 634 635 if len(stdout) > 0 { 636 sh := b.Shell(a) 637 if cfg.BuildX || cfg.BuildN { 638 sh.ShowCmd("", "%s # internal", joinUnambiguously(str.StringList("cat", c.OutputFile(stdoutEntry.OutputID)))) 639 } 640 if !cfg.BuildN { 641 sh.Printf("%s", stdout) 642 } 643 } 644 return nil 645 } 646 647 // flushOutput flushes the output being queued in a. 648 func (b *Builder) flushOutput(a *Action) { 649 b.Shell(a).Printf("%s", a.output) 650 a.output = nil 651 } 652 653 // updateBuildID updates the build ID in the target written by action a. 654 // It requires that useCache was called for action a and returned false, 655 // and that the build was then carried out and given the temporary 656 // a.buildID to record as the build ID in the resulting package or binary. 657 // updateBuildID computes the final content ID and updates the build IDs 658 // in the binary. 659 // 660 // Keep in sync with src/cmd/buildid/buildid.go 661 func (b *Builder) updateBuildID(a *Action, target string) error { 662 sh := b.Shell(a) 663 664 if cfg.BuildX || cfg.BuildN { 665 sh.ShowCmd("", "%s # internal", joinUnambiguously(str.StringList("go", "tool", "buildid", "-w", target))) 666 if cfg.BuildN { 667 return nil 668 } 669 } 670 671 c := cache.Default() 672 673 // Cache output from compile/link, even if we don't do the rest. 674 switch a.Mode { 675 case "build": 676 cache.PutBytes(c, cache.Subkey(a.actionID, "stdout"), a.output) 677 case "link": 678 // Even though we don't cache the binary, cache the linker text output. 679 // We might notice that an installed binary is up-to-date but still 680 // want to pretend to have run the linker. 681 // Store it under the main package's action ID 682 // to make it easier to find when that's all we have. 683 for _, a1 := range a.Deps { 684 if p1 := a1.Package; p1 != nil && p1.Name == "main" { 685 cache.PutBytes(c, cache.Subkey(a1.actionID, "link-stdout"), a.output) 686 break 687 } 688 } 689 } 690 691 // Find occurrences of old ID and compute new content-based ID. 692 r, err := os.Open(target) 693 if err != nil { 694 return err 695 } 696 matches, hash, err := buildid.FindAndHash(r, a.buildID, 0) 697 r.Close() 698 if err != nil { 699 return err 700 } 701 newID := a.buildID[:strings.LastIndex(a.buildID, buildIDSeparator)] + buildIDSeparator + buildid.HashToString(hash) 702 if len(newID) != len(a.buildID) { 703 return fmt.Errorf("internal error: build ID length mismatch %q vs %q", a.buildID, newID) 704 } 705 706 // Replace with new content-based ID. 707 a.buildID = newID 708 if a.json != nil { 709 a.json.BuildID = a.buildID 710 } 711 if len(matches) == 0 { 712 // Assume the user specified -buildid= to override what we were going to choose. 713 return nil 714 } 715 716 // Replace the build id in the file with the content-based ID. 717 w, err := os.OpenFile(target, os.O_RDWR, 0) 718 if err != nil { 719 return err 720 } 721 err = buildid.Rewrite(w, matches, newID) 722 if err != nil { 723 w.Close() 724 return err 725 } 726 if err := w.Close(); err != nil { 727 return err 728 } 729 730 // Cache package builds, and cache executable builds if 731 // executable caching was requested. Executables are not 732 // cached by default because they are not reused 733 // nearly as often as individual packages, and they're 734 // much larger, so the cache-footprint-to-utility ratio 735 // of executables is much lower for executables. 736 if a.Mode == "build" { 737 r, err := os.Open(target) 738 if err == nil { 739 if a.output == nil { 740 panic("internal error: a.output not set") 741 } 742 outputID, _, err := c.Put(a.actionID, r) 743 r.Close() 744 if err == nil && cfg.BuildX { 745 sh.ShowCmd("", "%s # internal", joinUnambiguously(str.StringList("cp", target, c.OutputFile(outputID)))) 746 } 747 if b.NeedExport { 748 if err != nil { 749 return err 750 } 751 a.Package.Export = c.OutputFile(outputID) 752 a.Package.BuildID = a.buildID 753 } 754 } 755 } 756 if c, ok := c.(*cache.DiskCache); a.Mode == "link" && a.CacheExecutable && ok { 757 r, err := os.Open(target) 758 if err == nil { 759 if a.output == nil { 760 panic("internal error: a.output not set") 761 } 762 name := a.Package.Internal.ExeName 763 if name == "" { 764 name = a.Package.DefaultExecName() 765 } 766 outputID, _, err := c.PutExecutable(a.actionID, name+cfg.ExeSuffix, r) 767 r.Close() 768 a.cachedExecutable = c.OutputFile(outputID) 769 if err == nil && cfg.BuildX { 770 sh.ShowCmd("", "%s # internal", joinUnambiguously(str.StringList("cp", target, a.cachedExecutable))) 771 } 772 } 773 } 774 775 return nil 776 } 777