metamerge.go

     1  // Copyright 2022 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  // This file contains functions and apis that support merging of
     8  // meta-data information.  It helps implement the "merge", "subtract",
     9  // and "intersect" subcommands.
    10  
    11  import (
    12  	"fmt"
    13  	"hash/fnv"
    14  	"internal/coverage"
    15  	"internal/coverage/calloc"
    16  	"internal/coverage/cmerge"
    17  	"internal/coverage/decodecounter"
    18  	"internal/coverage/decodemeta"
    19  	"internal/coverage/encodecounter"
    20  	"internal/coverage/encodemeta"
    21  	"internal/coverage/slicewriter"
    22  	"io"
    23  	"os"
    24  	"path/filepath"
    25  	"sort"
    26  	"time"
    27  	"unsafe"
    28  )
    29  
    30  // metaMerge provides state and methods to help manage the process
    31  // of selecting or merging meta data files. There are three cases
    32  // of interest here: the "-pcombine" flag provided by merge, the
    33  // "-pkg" option provided by all merge/subtract/intersect, and
    34  // a regular vanilla merge with no package selection
    35  //
    36  // In the -pcombine case, we're essentially glomming together all the
    37  // meta-data for all packages and all functions, meaning that
    38  // everything we see in a given package needs to be added into the
    39  // meta-data file builder; we emit a single meta-data file at the end
    40  // of the run.
    41  //
    42  // In the -pkg case, we will typically emit a single meta-data file
    43  // per input pod, where that new meta-data file contains entries for
    44  // just the selected packages.
    45  //
    46  // In the third case (vanilla merge with no combining or package
    47  // selection) we can carry over meta-data files without touching them
    48  // at all (only counter data files will be merged).
    49  type metaMerge struct {
    50  	calloc.BatchCounterAlloc
    51  	cmerge.Merger
    52  	// maps package import path to package state
    53  	pkm map[string]*pkstate
    54  	// list of packages
    55  	pkgs []*pkstate
    56  	// current package state
    57  	p *pkstate
    58  	// current pod state
    59  	pod *podstate
    60  	// counter data file osargs/goos/goarch state
    61  	astate *argstate
    62  }
    63  
    64  // pkstate
    65  type pkstate struct {
    66  	// index of package within meta-data file.
    67  	pkgIdx uint32
    68  	// this maps function index within the package to counter data payload
    69  	ctab map[uint32]decodecounter.FuncPayload
    70  	// pointer to meta-data blob for package
    71  	mdblob []byte
    72  	// filled in only for -pcombine merges
    73  	*pcombinestate
    74  }
    75  
    76  type podstate struct {
    77  	pmm      map[pkfunc]decodecounter.FuncPayload
    78  	mdf      string
    79  	mfr      *decodemeta.CoverageMetaFileReader
    80  	fileHash [16]byte
    81  }
    82  
    83  type pkfunc struct {
    84  	pk, fcn uint32
    85  }
    86  
    87  // pcombinestate
    88  type pcombinestate struct {
    89  	// Meta-data builder for the package.
    90  	cmdb *encodemeta.CoverageMetaDataBuilder
    91  	// Maps function meta-data hash to new function index in the
    92  	// new version of the package we're building.
    93  	ftab map[[16]byte]uint32
    94  }
    95  
    96  func newMetaMerge() *metaMerge {
    97  	return &metaMerge{
    98  		pkm:    make(map[string]*pkstate),
    99  		astate: &argstate{},
   100  	}
   101  }
   102  
   103  func (mm *metaMerge) visitMetaDataFile(mdf string, mfr *decodemeta.CoverageMetaFileReader) {
   104  	dbgtrace(2, "visitMetaDataFile(mdf=%s)", mdf)
   105  
   106  	// Record meta-data file name.
   107  	mm.pod.mdf = mdf
   108  	// Keep a pointer to the file-level reader.
   109  	mm.pod.mfr = mfr
   110  	// Record file hash.
   111  	mm.pod.fileHash = mfr.FileHash()
   112  	// Counter mode and granularity -- detect and record clashes here.
   113  	newgran := mfr.CounterGranularity()
   114  	newmode := mfr.CounterMode()
   115  	if err := mm.SetModeAndGranularity(mdf, newmode, newgran); err != nil {
   116  		fatal("%v", err)
   117  	}
   118  }
   119  
   120  func (mm *metaMerge) beginCounterDataFile(cdr *decodecounter.CounterDataReader) {
   121  	state := argvalues{
   122  		osargs: cdr.OsArgs(),
   123  		goos:   cdr.Goos(),
   124  		goarch: cdr.Goarch(),
   125  	}
   126  	mm.astate.Merge(state)
   127  }
   128  
   129  func copyMetaDataFile(inpath, outpath string) {
   130  	inf, err := os.Open(inpath)
   131  	if err != nil {
   132  		fatal("opening input meta-data file %s: %v", inpath, err)
   133  	}
   134  	defer inf.Close()
   135  
   136  	fi, err := inf.Stat()
   137  	if err != nil {
   138  		fatal("accessing input meta-data file %s: %v", inpath, err)
   139  	}
   140  
   141  	outf, err := os.OpenFile(outpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fi.Mode())
   142  	if err != nil {
   143  		fatal("opening output meta-data file %s: %v", outpath, err)
   144  	}
   145  
   146  	_, err = io.Copy(outf, inf)
   147  	outf.Close()
   148  	if err != nil {
   149  		fatal("writing output meta-data file %s: %v", outpath, err)
   150  	}
   151  }
   152  
   153  func (mm *metaMerge) beginPod() {
   154  	mm.pod = &podstate{
   155  		pmm: make(map[pkfunc]decodecounter.FuncPayload),
   156  	}
   157  }
   158  
   159  // metaEndPod handles actions needed when we're done visiting all of
   160  // the things in a pod -- counter files and meta-data file. There are
   161  // three cases of interest here:
   162  //
   163  // Case 1: in an unconditional merge (we're not selecting a specific set of
   164  // packages using "-pkg", and the "-pcombine" option is not in use),
   165  // we can simply copy over the meta-data file from input to output.
   166  //
   167  // Case 2: if this is a select merge (-pkg is in effect), then at
   168  // this point we write out a new smaller meta-data file that includes
   169  // only the packages of interest. At this point we also emit a merged
   170  // counter data file as well.
   171  //
   172  // Case 3: if "-pcombine" is in effect, we don't write anything at
   173  // this point (all writes will happen at the end of the run).
   174  func (mm *metaMerge) endPod(pcombine bool) {
   175  	if pcombine {
   176  		// Just clear out the pod data, we'll do all the
   177  		// heavy lifting at the end.
   178  		mm.pod = nil
   179  		return
   180  	}
   181  
   182  	finalHash := mm.pod.fileHash
   183  	if matchpkg != nil {
   184  		// Emit modified meta-data file for this pod.
   185  		finalHash = mm.emitMeta(*outdirflag, pcombine)
   186  	} else {
   187  		// Copy meta-data file for this pod to the output directory.
   188  		inpath := mm.pod.mdf
   189  		mdfbase := filepath.Base(mm.pod.mdf)
   190  		outpath := filepath.Join(*outdirflag, mdfbase)
   191  		copyMetaDataFile(inpath, outpath)
   192  	}
   193  
   194  	// Emit accumulated counter data for this pod.
   195  	mm.emitCounters(*outdirflag, finalHash)
   196  
   197  	// Reset package state.
   198  	mm.pkm = make(map[string]*pkstate)
   199  	mm.pkgs = nil
   200  	mm.pod = nil
   201  
   202  	// Reset counter mode and granularity
   203  	mm.ResetModeAndGranularity()
   204  }
   205  
   206  // emitMeta encodes and writes out a new coverage meta-data file as
   207  // part of a merge operation, specifically a merge with the
   208  // "-pcombine" flag.
   209  func (mm *metaMerge) emitMeta(outdir string, pcombine bool) [16]byte {
   210  	fh := fnv.New128a()
   211  	fhSum := fnv.New128a()
   212  	blobs := [][]byte{}
   213  	tlen := uint64(unsafe.Sizeof(coverage.MetaFileHeader{}))
   214  	for _, p := range mm.pkgs {
   215  		var blob []byte
   216  		if pcombine {
   217  			mdw := &slicewriter.WriteSeeker{}
   218  			p.cmdb.Emit(mdw)
   219  			blob = mdw.BytesWritten()
   220  		} else {
   221  			blob = p.mdblob
   222  		}
   223  		fhSum.Reset()
   224  		fhSum.Write(blob)
   225  		ph := fhSum.Sum(nil)
   226  		blobs = append(blobs, blob)
   227  		if _, err := fh.Write(ph[:]); err != nil {
   228  			panic(fmt.Sprintf("internal error: md5 sum failed: %v", err))
   229  		}
   230  		tlen += uint64(len(blob))
   231  	}
   232  	var finalHash [16]byte
   233  	fhh := fh.Sum(nil)
   234  	copy(finalHash[:], fhh)
   235  
   236  	// Open meta-file for writing.
   237  	fn := fmt.Sprintf("%s.%x", coverage.MetaFilePref, finalHash)
   238  	fpath := filepath.Join(outdir, fn)
   239  	mf, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
   240  	if err != nil {
   241  		fatal("unable to open output meta-data file %s: %v", fpath, err)
   242  	}
   243  
   244  	defer func() {
   245  		if err := mf.Close(); err != nil {
   246  			fatal("error closing output meta-data file %s: %v", fpath, err)
   247  		}
   248  	}()
   249  
   250  	// Encode and write.
   251  	mfw := encodemeta.NewCoverageMetaFileWriter(fpath, mf)
   252  	err = mfw.Write(finalHash, blobs, mm.Mode(), mm.Granularity())
   253  	if err != nil {
   254  		fatal("error writing %s: %v\n", fpath, err)
   255  	}
   256  	return finalHash
   257  }
   258  
   259  func (mm *metaMerge) emitCounters(outdir string, metaHash [16]byte) {
   260  	// Open output file. The file naming scheme is intended to mimic
   261  	// that used when running a coverage-instrumented binary, for
   262  	// consistency (however the process ID is not meaningful here, so
   263  	// use a value of zero).
   264  	var dummyPID int
   265  	fn := fmt.Sprintf(coverage.CounterFileTempl, coverage.CounterFilePref, metaHash, dummyPID, time.Now().UnixNano())
   266  	fpath := filepath.Join(outdir, fn)
   267  	cf, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
   268  	if err != nil {
   269  		fatal("opening counter data file %s: %v", fpath, err)
   270  	}
   271  	defer func() {
   272  		if err := cf.Close(); err != nil {
   273  			fatal("error closing output meta-data file %s: %v", fpath, err)
   274  		}
   275  	}()
   276  
   277  	args := mm.astate.ArgsSummary()
   278  	cfw := encodecounter.NewCoverageDataWriter(cf, coverage.CtrULeb128)
   279  	if err := cfw.Write(metaHash, args, mm); err != nil {
   280  		fatal("counter file write failed: %v", err)
   281  	}
   282  	mm.astate = &argstate{}
   283  }
   284  
   285  // VisitFuncs is used while writing the counter data files; it
   286  // implements the 'VisitFuncs' method required by the interface
   287  // internal/coverage/encodecounter/CounterVisitor.
   288  func (mm *metaMerge) VisitFuncs(f encodecounter.CounterVisitorFn) error {
   289  	if *verbflag >= 4 {
   290  		fmt.Printf("counterVisitor invoked\n")
   291  	}
   292  	// For each package, for each function, construct counter
   293  	// array and then call "f" on it.
   294  	for pidx, p := range mm.pkgs {
   295  		fids := make([]int, 0, len(p.ctab))
   296  		for fid := range p.ctab {
   297  			fids = append(fids, int(fid))
   298  		}
   299  		sort.Ints(fids)
   300  		if *verbflag >= 4 {
   301  			fmt.Printf("fids for pk=%d: %+v\n", pidx, fids)
   302  		}
   303  		for _, fid := range fids {
   304  			fp := p.ctab[uint32(fid)]
   305  			if *verbflag >= 4 {
   306  				fmt.Printf("counter write for pk=%d fid=%d len(ctrs)=%d\n", pidx, fid, len(fp.Counters))
   307  			}
   308  			if err := f(uint32(pidx), uint32(fid), fp.Counters); err != nil {
   309  				return err
   310  			}
   311  		}
   312  	}
   313  	return nil
   314  }
   315  
   316  func (mm *metaMerge) visitPackage(pd *decodemeta.CoverageMetaDataDecoder, pkgIdx uint32, pcombine bool) {
   317  	p, ok := mm.pkm[pd.PackagePath()]
   318  	if !ok {
   319  		p = &pkstate{
   320  			pkgIdx: uint32(len(mm.pkgs)),
   321  		}
   322  		mm.pkgs = append(mm.pkgs, p)
   323  		mm.pkm[pd.PackagePath()] = p
   324  		if pcombine {
   325  			p.pcombinestate = new(pcombinestate)
   326  			cmdb, err := encodemeta.NewCoverageMetaDataBuilder(pd.PackagePath(), pd.PackageName(), pd.ModulePath())
   327  			if err != nil {
   328  				fatal("fatal error creating meta-data builder: %v", err)
   329  			}
   330  			dbgtrace(2, "install new pkm entry for package %s pk=%d", pd.PackagePath(), pkgIdx)
   331  			p.cmdb = cmdb
   332  			p.ftab = make(map[[16]byte]uint32)
   333  		} else {
   334  			var err error
   335  			p.mdblob, err = mm.pod.mfr.GetPackagePayload(pkgIdx, nil)
   336  			if err != nil {
   337  				fatal("error extracting package %d payload from %s: %v",
   338  					pkgIdx, mm.pod.mdf, err)
   339  			}
   340  		}
   341  		p.ctab = make(map[uint32]decodecounter.FuncPayload)
   342  	}
   343  	mm.p = p
   344  }
   345  
   346  func (mm *metaMerge) visitFuncCounterData(data decodecounter.FuncPayload) {
   347  	key := pkfunc{pk: data.PkgIdx, fcn: data.FuncIdx}
   348  	val := mm.pod.pmm[key]
   349  	// FIXME: in theory either A) len(val.Counters) is zero, or B)
   350  	// the two lengths are equal. Assert if not? Of course, we could
   351  	// see odd stuff if there is source file skew.
   352  	if *verbflag > 4 {
   353  		fmt.Printf("visit pk=%d fid=%d len(counters)=%d\n", data.PkgIdx, data.FuncIdx, len(data.Counters))
   354  	}
   355  	if len(val.Counters) < len(data.Counters) {
   356  		t := val.Counters
   357  		val.Counters = mm.AllocateCounters(len(data.Counters))
   358  		copy(val.Counters, t)
   359  	}
   360  	err, overflow := mm.MergeCounters(val.Counters, data.Counters)
   361  	if err != nil {
   362  		fatal("%v", err)
   363  	}
   364  	if overflow {
   365  		warn("uint32 overflow during counter merge")
   366  	}
   367  	mm.pod.pmm[key] = val
   368  }
   369  
   370  func (mm *metaMerge) visitFunc(pkgIdx uint32, fnIdx uint32, fd *coverage.FuncDesc, verb string, pcombine bool) {
   371  	if *verbflag >= 3 {
   372  		fmt.Printf("visit pk=%d fid=%d func %s\n", pkgIdx, fnIdx, fd.Funcname)
   373  	}
   374  
   375  	var counters []uint32
   376  	key := pkfunc{pk: pkgIdx, fcn: fnIdx}
   377  	v, haveCounters := mm.pod.pmm[key]
   378  	if haveCounters {
   379  		counters = v.Counters
   380  	}
   381  
   382  	if pcombine {
   383  		// If the merge is running in "combine programs" mode, then hash
   384  		// the function and look it up in the package ftab to see if we've
   385  		// encountered it before. If we haven't, then register it with the
   386  		// meta-data builder.
   387  		fnhash := encodemeta.HashFuncDesc(fd)
   388  		gfidx, ok := mm.p.ftab[fnhash]
   389  		if !ok {
   390  			// We haven't seen this function before, need to add it to
   391  			// the meta data.
   392  			gfidx = uint32(mm.p.cmdb.AddFunc(*fd))
   393  			mm.p.ftab[fnhash] = gfidx
   394  			if *verbflag >= 3 {
   395  				fmt.Printf("new meta entry for fn %s fid=%d\n", fd.Funcname, gfidx)
   396  			}
   397  		}
   398  		fnIdx = gfidx
   399  	}
   400  	if !haveCounters {
   401  		return
   402  	}
   403  
   404  	// Install counters in package ctab.
   405  	gfp, ok := mm.p.ctab[fnIdx]
   406  	if ok {
   407  		if verb == "subtract" || verb == "intersect" {
   408  			panic("should never see this for intersect/subtract")
   409  		}
   410  		if *verbflag >= 3 {
   411  			fmt.Printf("counter merge for %s fidx=%d\n", fd.Funcname, fnIdx)
   412  		}
   413  		// Merge.
   414  		err, overflow := mm.MergeCounters(gfp.Counters, counters)
   415  		if err != nil {
   416  			fatal("%v", err)
   417  		}
   418  		if overflow {
   419  			warn("uint32 overflow during counter merge")
   420  		}
   421  		mm.p.ctab[fnIdx] = gfp
   422  	} else {
   423  		if *verbflag >= 3 {
   424  			fmt.Printf("null merge for %s fidx %d\n", fd.Funcname, fnIdx)
   425  		}
   426  		gfp := v
   427  		gfp.PkgIdx = mm.p.pkgIdx
   428  		gfp.FuncIdx = fnIdx
   429  		mm.p.ctab[fnIdx] = gfp
   430  	}
   431  }
   432
View as plain text