Improve Go module support

This commit is contained in:
Trevor Slocum 2020-09-10 15:44:49 -07:00
parent a169d7aa4a
commit 25611b612c
6 changed files with 225 additions and 91 deletions

View File

@ -1,3 +1,8 @@
0.1.8:
- Improve Go module support
- Filter internal, testdata and cmd packages from docs
- Fix Windows support
0.1.7:
- Print godoc installation instructions when it is not found

View File

@ -25,20 +25,22 @@ go get golang.org/x/tools/cmd/godoc
## Documentation
To generate documentation for specific packages, execute `godoc-static`
supplying at least one package name:
supplying at least one package import path and/or absolute path:
```bash
godoc-static -destination=/home/user/sites/docs fmt net/http
godoc-static -destination=/home/user/sites/docs fmt net/http ~/awesomeproject
```
When an import path is supplied, the package is sourced from `$GOPATH` or `$GOROOT`.
When no packages are supplied, documentation is generated for packages listed
by `go list ...` instead.
by `go list ...`.
Packages are not downloaded/updated automatically.
### Usage examples
Generate documentation for `archive`, `net/http` and `gitlab.com/tslocum/cview`:
Generate documentation for `archive`, `net/http` and `~/go/src/gitlab.com/tslocum/cview`:
```bash
godoc-static \

9
go.mod
View File

@ -1,9 +1,12 @@
module gitlab.com/tslocum/godoc-static
go 1.13
go 1.15
require (
github.com/PuerkitoBio/goquery v1.5.1
github.com/yuin/goldmark v1.1.23
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b
github.com/andybalholm/cascadia v1.2.0 // indirect
github.com/yuin/goldmark v1.2.1
golang.org/x/mod v0.3.0
golang.org/x/net v0.0.0-20200904194848-62affa334b73
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
)

26
go.sum
View File

@ -2,13 +2,29 @@ github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154Oa
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/yuin/goldmark v1.1.23 h1:eTodJ8hwEUvwXhb9qxQNuL/q1d+xMQClrXR4mdvV7gs=
github.com/yuin/goldmark v1.1.23/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

265
main.go
View File

@ -7,6 +7,7 @@ import (
"errors"
"flag"
"fmt"
"go/build"
"io/ioutil"
"log"
"net/http"
@ -23,6 +24,7 @@ import (
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/extension"
gmhtml "github.com/yuin/goldmark/renderer/html"
"golang.org/x/mod/modfile"
"golang.org/x/net/html"
)
@ -39,8 +41,14 @@ var (
excludePackages string
verbose bool
godoc *exec.Cmd
outZip *zip.Writer
goPath string
godoc *exec.Cmd
godocEnv []string
godocStartDir string
outZip *zip.Writer
scanIncomplete = []byte(`<span class="alert" style="font-size:120%">Scan is not yet complete.`)
)
func main() {
@ -117,9 +125,42 @@ func writeFile(buf *bytes.Buffer, fileDir string, fileName string) error {
return ioutil.WriteFile(path.Join(siteDestination, fileDir, fileName), buf.Bytes(), 0755)
}
func startGodoc(dir string) {
if dir == godocStartDir {
return // Already started
}
godocStartDir = dir
if godoc != nil {
godoc.Process.Kill()
godoc.Wait()
}
godoc = exec.Command("godoc", fmt.Sprintf("-http=%s", listenAddress))
godoc.Env = godocEnv
if dir == "" {
godoc.Dir = os.TempDir()
} else {
godoc.Dir = dir
}
godoc.Stdin = nil
godoc.Stdout = nil
godoc.Stderr = nil
setDeathSignal(godoc)
err := godoc.Start()
if err != nil {
log.Fatalf("failed to execute godoc: %s\ninstall godoc by running: go get golang.org/x/tools/cmd/godoc\nthen ensure ~/go/bin is in $PATH", err)
}
}
func run() error {
var buf bytes.Buffer
timeStarted := time.Now()
var (
timeStarted = time.Now()
buf bytes.Buffer
err error
)
if siteDestination == "" {
return errors.New("--destination must be set")
@ -188,20 +229,22 @@ func run() error {
defer outZip.Close()
}
if verbose {
log.Println("Starting godoc...")
goPath = os.Getenv("GOPATH")
if goPath == "" {
goPath = build.Default.GOPATH
}
godoc = exec.Command("godoc", fmt.Sprintf("-http=%s", listenAddress))
godoc.Stdin = os.Stdin
godoc.Stdout = os.Stdout
godoc.Stderr = os.Stderr
setDeathSignal(godoc)
err := godoc.Start()
if err != nil {
return fmt.Errorf("failed to execute godoc: %s\ninstall godoc by running: go get golang.org/x/tools/cmd/godoc\nthen ensure ~/go/bin is in $PATH", err)
godocEnv = make([]string, len(os.Environ()))
copy(godocEnv, os.Environ())
for i, e := range godocEnv {
if strings.HasPrefix(e, "GO111MODULE=") {
godocEnv[i] = ""
}
}
godocEnv = append(godocEnv, "GO111MODULE=auto")
godocStartDir = "-" // Trigger initial start
startGodoc("")
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
@ -211,14 +254,13 @@ func run() error {
os.Exit(1)
}()
godocStarted := time.Now()
pkgs := flag.Args()
if len(pkgs) == 0 || (len(pkgs) == 1 && pkgs[0] == "") {
buf.Reset()
cmd := exec.Command("go", "list", "...")
cmd.Env = godocEnv
cmd.Dir = os.TempDir()
cmd.Stdout = &buf
setDeathSignal(cmd)
@ -232,43 +274,84 @@ func run() error {
}
var newPkgs []string
pkgPaths := make(map[string]string)
for _, pkg := range pkgs {
if strings.TrimSpace(pkg) == "" {
continue
}
buf.Reset()
var suppliedPath bool
dir := ""
if _, err := os.Stat(pkg); !os.IsNotExist(err) {
dir = pkg
modFileData, err := ioutil.ReadFile(path.Join(dir, "go.mod"))
if err != nil {
log.Fatalf("failed to read mod file for %s: %s", pkg, err)
}
modFile, err := modfile.Parse(path.Join(dir, "go.mod"), modFileData, nil)
if err != nil {
log.Fatalf("failed to parse mod file for %s: %s", pkg, err)
}
pkg = modFile.Module.Mod.Path
suppliedPath = true
} else {
srcDir := path.Join(goPath, "src", pkg)
if _, err := os.Stat(srcDir); !os.IsNotExist(err) {
dir = srcDir
}
}
newPkgs = append(newPkgs, pkg)
cmd := exec.Command("go", "list", "-find", "-f", `{{ .Dir }}`, pkg)
cmd.Dir = os.TempDir()
buf.Reset()
search := "./..."
if dir == "" {
search = pkg
}
cmd := exec.Command("go", "list", "-find", "-f", `{{ .ImportPath }} {{ .Dir }}`, search)
cmd.Env = godocEnv
if dir == "" {
cmd.Dir = os.TempDir()
} else {
cmd.Dir = dir
}
cmd.Stdout = &buf
cmd.Stderr = &buf
setDeathSignal(cmd)
err = cmd.Run()
if err != nil {
return fmt.Errorf("failed to list source directory of package %s: %s", pkg, err)
pkgPaths[pkg] = dir
continue
}
pkgPath := strings.TrimSpace(buf.String())
if pkgPath != "" {
err := filepath.Walk(pkgPath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
} else if !info.IsDir() {
return nil
} else if strings.HasPrefix(filepath.Base(path), ".") {
return filepath.SkipDir
}
sourceListing := strings.Split(buf.String(), "\n")
for i := range sourceListing {
firstSpace := strings.Index(sourceListing[i], " ")
if firstSpace <= 0 {
continue
}
if len(path) > len(pkgPath) && strings.HasPrefix(path, pkgPath) {
newPkgs = append(newPkgs, pkg+path[len(pkgPath):])
}
return nil
})
if err != nil {
return fmt.Errorf("failed to walk source directory of package %s: %s", pkg, err)
pkg = sourceListing[i][:firstSpace]
pkgPath := sourceListing[i][firstSpace+1:]
newPkgs = append(newPkgs, pkg)
if dir == "" || strings.HasPrefix(filepath.Base(pkgPath), ".") {
continue
}
if suppliedPath {
pkgPaths[pkg] = dir
} else {
pkgPaths[pkg] = pkgPath
}
}
buf.Reset()
@ -293,40 +376,46 @@ func run() error {
return strings.ToLower(pkgs[i]) < strings.ToLower(pkgs[j])
})
// Allow some time for godoc to initialize
if time.Since(godocStarted) < 3*time.Second {
time.Sleep((3 * time.Second) - time.Since(godocStarted))
}
done := make(chan error)
timeout := time.After(15 * time.Second)
go func() {
var (
res *http.Response
doc *goquery.Document
err error
)
for _, pkg := range pkgs {
for _, pkg := range filterPkgs {
if verbose {
log.Printf("Copying %s docs...", pkg)
log.Printf("Copying %s documentation...", pkg)
}
startGodoc(pkgPaths[pkg])
// Rely on timeout to break loop
for {
res, err = http.Get(fmt.Sprintf("http://%s/pkg/%s/", listenAddress, pkg))
if err == nil {
body, err := ioutil.ReadAll(res.Body)
if err != nil {
done <- fmt.Errorf("failed to get page of %s: %s", pkg, err)
return
}
if bytes.Contains(body, scanIncomplete) {
time.Sleep(25 * time.Millisecond)
continue
}
// Load the HTML document
doc, err = goquery.NewDocumentFromReader(bytes.NewReader(body))
if err != nil {
done <- fmt.Errorf("failed to parse page of %s: %s", pkg, err)
return
}
break
}
}
// Load the HTML document
doc, err := goquery.NewDocumentFromReader(res.Body)
if err != nil {
done <- fmt.Errorf("failed to get page of %s: %s", pkg, err)
return
}
doc.Find("title").First().SetHtml(fmt.Sprintf("%s - %s", path.Base(pkg), siteName))
updatePage(doc, relativeBasePath(pkg), siteName)
@ -355,8 +444,6 @@ func run() error {
}()
select {
case <-timeout:
return errors.New("godoc failed to start in time")
case err = <-done:
if err != nil {
return fmt.Errorf("failed to copy docs: %s", err)
@ -370,8 +457,6 @@ func run() error {
return fmt.Errorf("failed to make directory lib: %s", err)
}
filterPkgs = filterPkgsWithExcludes(filterPkgs)
for _, pkg := range filterPkgs {
if verbose {
log.Printf("Copying %s sources...", pkg)
@ -379,6 +464,13 @@ func run() error {
buf.Reset()
dir := pkgPaths[pkg]
if dir == "" {
dir = getTmpDir()
}
startGodoc(pkgPaths[pkg])
cmd := exec.Command("go", "list", "-find", "-f",
`{{ join .GoFiles "\n" }}`+"\n"+
`{{ join .CgoFiles "\n" }}`+"\n"+
@ -393,7 +485,8 @@ func run() error {
`{{ join .TestGoFiles "\n" }}`+"\n"+
`{{ join .XTestGoFiles "\n" }}`,
pkg)
cmd.Dir = getTmpDir()
cmd.Env = godocEnv
cmd.Dir = dir
cmd.Stdout = &buf
setDeathSignal(cmd)
@ -411,15 +504,28 @@ func run() error {
}
// Rely on timeout to break loop
res, err := http.Get(fmt.Sprintf("http://%s/src/%s/%s", listenAddress, pkg, sourceFile))
if err != nil {
return fmt.Errorf("failed to get source file page %s for package %s: %s", sourceFile, pkg, err)
}
var doc *goquery.Document
for {
res, err := http.Get(fmt.Sprintf("http://%s/src/%s/%s", listenAddress, pkg, sourceFile))
if err == nil {
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return fmt.Errorf("failed to get source file page %s of %s: %s", sourceFile, pkg, err)
}
// Load the HTML document
doc, err := goquery.NewDocumentFromReader(res.Body)
if err != nil {
return fmt.Errorf("failed to load document from page for package %s: %s", pkg, err)
if bytes.Contains(body, scanIncomplete) {
time.Sleep(25 * time.Millisecond)
continue
}
// Load the HTML document
doc, err = goquery.NewDocumentFromReader(bytes.NewReader(body))
if err != nil {
return fmt.Errorf("failed to load document from page for package %s: %s", pkg, err)
}
break
}
}
doc.Find("title").First().SetHtml(fmt.Sprintf("%s - %s", path.Base(pkg), siteName))
@ -468,17 +574,18 @@ func run() error {
return fmt.Errorf("failed to make directory lib: %s", err)
}
res, err := http.Get(fmt.Sprintf("http://%s/lib/godoc/style.css", listenAddress))
if err != nil {
return fmt.Errorf("failed to get syle.css: %s", err)
}
for {
res, err := http.Get(fmt.Sprintf("http://%s/lib/godoc/style.css", listenAddress))
if err == nil {
buf.Reset()
buf.Reset()
_, err = buf.ReadFrom(res.Body)
res.Body.Close()
if err != nil {
return fmt.Errorf("failed to get style.css: %s", err)
_, err = buf.ReadFrom(res.Body)
res.Body.Close()
if err != nil {
return fmt.Errorf("failed to get style.css: %s", err)
}
break
}
}
buf.WriteString("\n" + additionalCSS)
@ -491,7 +598,7 @@ func run() error {
// Write index
if verbose {
log.Println("Writing index...")
log.Println("Writing index.html...")
}
err = writeIndex(&buf, pkgs, filterPkgs)

View File

@ -227,6 +227,7 @@ func writeIndex(buf *bytes.Buffer, pkgs []string, filterPkgs []string) error {
for _, pkg := range pkgs {
pkgBuf.Reset()
cmd := exec.Command("go", "list", "-find", "-f", `{{ .Doc }}`, pkg)
cmd.Env = godocEnv
cmd.Dir = os.TempDir()
cmd.Stdout = &pkgBuf
setDeathSignal(cmd)