From 8d6cb6527e9f90ca0d120dd856b2bfe8bd6b73d4 Mon Sep 17 00:00:00 2001 From: Trevor Slocum Date: Thu, 3 Dec 2020 11:12:13 -0800 Subject: [PATCH] Serve Gemini content via HTTPS --- CONFIGURATION.md | 84 ++++++++++--------- README.md | 4 +- config.go | 77 +++++++++-------- go.mod | 3 +- go.sum | 6 +- serve_fcgi.go | 12 +-- serve_https.go | 212 +++++++++++++++++++++++++++++++++++++++++++++++ server.go | 85 ++++++++++++++----- 8 files changed, 372 insertions(+), 111 deletions(-) create mode 100644 serve_https.go diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 09efcfb..0abb53c 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -71,11 +71,19 @@ certbot certonly --config-dir /home/www/certs \ Provide the path to the certificate file at `certs/live/$DOMAIN/fullchain.pem` and the private key file at `certs/live/$DOMAIN/privkey.pem` to twins. +## DisableHTTPS + +Pages are also available via HTTPS on the same port by default. +Set this option to `true` to disable this feature. + +Pages are converted automatically by [gmitohtml](https://gitlab.com/tslocum/gmitohtml). + ### DisableSize The size of the response body is included in the media type header by default. -Set this option to `true` to disable this feature. See [PROPOSALS.md](https://gitlab.com/tslocum/twins/blob/master/PROPOSALS.md) -for more information. +Set this option to `true` to disable this feature. + +See [PROPOSALS.md](https://gitlab.com/tslocum/twins/blob/master/PROPOSALS.md) for more information. ### Path @@ -115,42 +123,6 @@ Cache duration (in seconds). Set to `0` to disable caching entirely. This is an out-of-spec feature. See [PROPOSALS.md](https://gitlab.com/tslocum/twins/blob/master/PROPOSALS.md) for more information. -##### Hidden - -When enabled, hidden files and directories may be accessed. This attribute is -disabled by default. - -##### Input - -Request text input from user. - -##### SensitiveInput - -Request sensitive text input from the user. Text will not be shown as it is entered. - -##### List - -When enabled, directories without an index file will serve a list of their -contents. This attribute is disabled by default. - -##### Lang - -Specifies content language. This is sent to clients via the MIME type `lang` parameter. - -##### Log - -Path to log file. Requests are logged in [Apache format](https://httpd.apache.org/docs/2.2/logs.html#combined), -excluding IP address and query. - -##### SymLinks - -When enabled, symbolic links may be accessed. This attribute is disabled by default. - -##### Type - -Content type is normally detected automatically. This attribute forces a -specific content type for a path. - ##### FastCGI Forward requests to [FastCGI](https://en.wikipedia.org/wiki/FastCGI) server at @@ -170,6 +142,42 @@ Connect via TCP: tcp://127.0.0.1:9000 ``` +##### Hidden + +When enabled, hidden files and directories may be accessed. This attribute is +disabled by default. + +##### Input + +Request text input from user. + +##### Lang + +Specifies content language. This is sent to clients via the MIME type `lang` parameter. + +##### List + +When enabled, directories without an index file will serve a list of their +contents. This attribute is disabled by default. + +##### Log + +Path to log file. Requests are logged in [Apache format](https://httpd.apache.org/docs/2.2/logs.html#combined), +excluding IP address and query. + +##### SensitiveInput + +Request sensitive text input from the user. Text will not be shown as it is entered. + +##### SymLinks + +When enabled, symbolic links may be accessed. This attribute is disabled by default. + +##### Type + +Content type is normally detected automatically. This attribute forces a +specific content type for a path. + ## End-of-line indicator The Gemini protocol requires `\r\n` (CRLF) as the end-of-line indicator. This diff --git a/README.md b/README.md index dcfa2f1..a850a45 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,8 @@ This page is also available at [gemini://twins.rocketnine.space](gemini://twins. - Reverse proxy requests - TCP - [FastCGI](https://en.wikipedia.org/wiki/FastCGI) -- Serve system command output -- Redirect to path or URL +- Serve Gemini content via HTTPS + - Pages are converted automatically by [gmitohtml](https://gitlab.com/tslocum/gmitohtml) - Reload configuration on `SIGHUP` ## Proposals diff --git a/config.go b/config.go index 8095a8f..4a75b4d 100644 --- a/config.go +++ b/config.go @@ -26,36 +26,36 @@ type pathConfig struct { Proxy string Redirect string + // Cache duration + Cache string + + // FastCGI server address + FastCGI string + + // Serve hidden files and directories + Hidden bool + // Request input Input string + // Language + Lang string + + // List directory + List bool + + // Log file + Log string + // Request sensitive input SensitiveInput string // Follow symbolic links SymLinks bool - // Serve hidden files and directories - Hidden bool - - // List directory - List bool - // Content type Type string - // Cache duration - Cache string - - // FastCGI server address - FastCGI string - - // Language - Lang string - - // Log file - Log string - r *regexp.Regexp cmd []string cache int64 @@ -70,15 +70,12 @@ type hostConfig struct { } type serverConfig struct { - Listen string - - Types map[string]string - - Hosts map[string]*hostConfig - - DisableSize bool - - SaneEOL bool + Listen string + Types map[string]string + Hosts map[string]*hostConfig + DisableHTTPS bool + DisableSize bool + SaneEOL bool hostname string port int @@ -167,36 +164,36 @@ func readconfig(configPath string) error { if len(defaultHost.Paths) == 1 { defaultPath := defaultHost.Paths[0] for _, serve := range host.Paths { + // Resources if defaultPath.Root != "" && serve.Root == "" { serve.Root = defaultPath.Root - } - if defaultPath.Command != "" && serve.Command == "" { + } else if defaultPath.Command != "" && serve.Command == "" { serve.Command = defaultPath.Command - } - if defaultPath.Proxy != "" && serve.Proxy == "" { + } else if defaultPath.Proxy != "" && serve.Proxy == "" { serve.Proxy = defaultPath.Proxy } - if defaultPath.SymLinks { - serve.SymLinks = defaultPath.SymLinks - } - if defaultPath.Hidden { - serve.Hidden = defaultPath.Hidden - } - if defaultPath.List { - serve.List = defaultPath.List - } + // Attributes if defaultPath.Cache != "" && serve.Cache == "" { serve.Cache = defaultPath.Cache } if defaultPath.FastCGI != "" && serve.FastCGI == "" { serve.FastCGI = defaultPath.FastCGI } + if defaultPath.Hidden { + serve.Hidden = defaultPath.Hidden + } if defaultPath.Lang != "" && serve.Lang == "" { serve.Lang = defaultPath.Lang } + if defaultPath.List { + serve.List = defaultPath.List + } if defaultPath.Log != "" && serve.Log == "" { serve.Log = defaultPath.Log } + if defaultPath.SymLinks { + serve.SymLinks = defaultPath.SymLinks + } } } else if len(defaultHost.Paths) > 1 { log.Fatal("only one path may be defined for the default host") diff --git a/go.mod b/go.mod index e1d0db6..cf05d3d 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/h2non/filetype v1.1.0 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/yookoala/gofast v0.4.1-0.20201013050739-975113c54107 - golang.org/x/tools v0.0.0-20201117152513-9036a0f9af11 // indirect + gitlab.com/tslocum/gmitohtml v1.0.3-0.20201203184239-2a1abe8efe7c + golang.org/x/tools v0.0.0-20201203170353-bdde1628ed1d // indirect gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 ) diff --git a/go.sum b/go.sum index 2ece2a7..58c49c8 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/yookoala/gofast v0.4.1-0.20201013050739-975113c54107 h1:wfqP/vw5tHVeFQJbnmyXSi7E2ZeshpKhR4kuUR5B7yQ= github.com/yookoala/gofast v0.4.1-0.20201013050739-975113c54107/go.mod h1:OJU201Q6HCaE1cASckaTbMm3KB6e0cZxK0mgqfwOKvQ= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +gitlab.com/tslocum/gmitohtml v1.0.3-0.20201203184239-2a1abe8efe7c h1:hew6E9kPxaR/fipooKTLB+ARkpmI9ShtzYZFdKG7jzQ= +gitlab.com/tslocum/gmitohtml v1.0.3-0.20201203184239-2a1abe8efe7c/go.mod h1:+LFeUUQ6kcjFUR2y3XVr/4i8qLmZOUnM/gwsb9leHMk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -34,8 +36,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200908211811-12e1bf57a112/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201117152513-9036a0f9af11 h1:gqcmLJzeDSNhSzkyhJ4kxP6CtTimi/5hWFDGp0lFd1w= -golang.org/x/tools v0.0.0-20201117152513-9036a0f9af11/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201203170353-bdde1628ed1d h1:OuIGT9zWmMvqaajHPt4H4W1omjiKwkGpBw5ttuErmnw= +golang.org/x/tools v0.0.0-20201203170353-bdde1628ed1d/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/serve_fcgi.go b/serve_fcgi.go index 0e4ba7c..9d89f25 100644 --- a/serve_fcgi.go +++ b/serve_fcgi.go @@ -11,23 +11,23 @@ import ( "github.com/yookoala/gofast" ) -type responseWriter struct { +type fakeResponseWriter struct { io.WriteCloser header http.Header } -func newResponseWriter(out io.WriteCloser) *responseWriter { - return &responseWriter{ +func newFakeResponseWriter(out io.WriteCloser) *fakeResponseWriter { + return &fakeResponseWriter{ WriteCloser: out, header: make(http.Header), } } -func (w *responseWriter) Header() http.Header { +func (w *fakeResponseWriter) Header() http.Header { return w.header } -func (w *responseWriter) WriteHeader(statusCode int) { +func (w *fakeResponseWriter) WriteHeader(statusCode int) { // Do nothing } @@ -53,5 +53,5 @@ func serveFastCGI(c net.Conn, connFactory gofast.ConnFactory, u *url.URL, filePa gofast.NewFileEndpoint(filePath)(gofast.BasicSession), gofast.SimpleClientFactory(connFactory, 0), ). - ServeHTTP(newResponseWriter(c), r) + ServeHTTP(newFakeResponseWriter(c), r) } diff --git a/serve_https.go b/serve_https.go new file mode 100644 index 0000000..48cbffc --- /dev/null +++ b/serve_https.go @@ -0,0 +1,212 @@ +package main + +import ( + "crypto/tls" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "path" + "strings" + + "gitlab.com/tslocum/gmitohtml/pkg/gmitohtml" +) + +var cssBytes = []byte(gmitohtml.StyleCSS) + +func serveHTTPS(w http.ResponseWriter, r *http.Request) (int, int64, string) { + if r.URL.Path == "" { + // Redirect to / + u, err := url.Parse(r.URL.String()) + if err != nil { + status := http.StatusInternalServerError + http.Error(w, "Failed to parse URL", status) + return status, -1, "" + } + u.Path += "/" + + status := http.StatusTemporaryRedirect + http.Redirect(w, r, u.String(), status) + return status, -1, "" + } else if r.URL.Path == "/assets/style.css" { + status := http.StatusOK + w.Header().Set("Content-Type", cssType) + w.WriteHeader(status) + + w.Write(cssBytes) + return status, int64(len(cssBytes)), "" + } + + pathBytes := []byte(r.URL.Path) + strippedPath := r.URL.Path + if strippedPath[0] == '/' { + strippedPath = strippedPath[1:] + } + + if host, ok := config.Hosts[r.URL.Hostname()]; ok { + for _, serve := range host.Paths { + matchedRegexp := serve.r != nil && serve.r.Match(pathBytes) + matchedPrefix := serve.r == nil && strings.HasPrefix(r.URL.Path, serve.Path) + if !matchedRegexp && !matchedPrefix { + continue + } + + requireInput := serve.Input != "" || serve.SensitiveInput != "" + if r.URL.RawQuery == "" && requireInput { + if serve.SensitiveInput != "" { + // TODO + } + status := http.StatusInternalServerError + http.Error(w, "Gemini to HTML conversion is not supported for this page", status) + return status, -1, serve.Log + } + + if matchedRegexp || matchedPrefix { + if serve.Root == "" || serve.FastCGI != "" { + status := http.StatusInternalServerError + http.Error(w, "Gemini to HTML conversion is not supported for this page", status) + return status, -1, serve.Log + } + + var filePath string + if serve.Root != "" { + root := serve.Root + if root[len(root)-1] != '/' { + root += "/" + } + + requestSplit := strings.Split(r.URL.Path, "/") + + if !serve.SymLinks { + for i := 1; i < len(requestSplit); i++ { + info, err := os.Lstat(path.Join(root, strings.Join(requestSplit[1:i+1], "/"))) + if err != nil || info.Mode()&os.ModeSymlink == os.ModeSymlink { + http.NotFound(w, r) + return http.StatusNotFound, -1, serve.Log + } + } + } + + filePath = path.Join(root, strings.Join(requestSplit[1:], "/")) + } + + fi, err := os.Stat(filePath) + if err != nil { + http.NotFound(w, r) + return http.StatusNotFound, -1, serve.Log + } + + mode := fi.Mode() + hasTrailingSlash := len(r.URL.Path) > 0 && r.URL.Path[len(r.URL.Path)-1] == '/' + if mode.IsDir() { + if !hasTrailingSlash { + u, err := url.Parse(r.URL.String()) + if err != nil { + status := http.StatusInternalServerError + http.Error(w, "Failed to parse URL", status) + return status, -1, serve.Log + } + u.Path += "/" + + status := http.StatusTemporaryRedirect + http.Redirect(w, r, u.String(), status) + return status, -1, serve.Log + } + + _, err := os.Stat(path.Join(filePath, "index.gmi")) + if err != nil { + _, err := os.Stat(path.Join(filePath, "index.gemini")) + if err != nil { + if serve.List { + status := http.StatusInternalServerError + http.Error(w, "HTTPS dir lost not yet implemented", status) + return status, -1, serve.Log + } + + http.NotFound(w, r) + return http.StatusNotFound, -1, serve.Log + } + filePath = path.Join(filePath, "index.gemini") + } else { + filePath = path.Join(filePath, "index.gmi") + } + } else if hasTrailingSlash && len(r.URL.Path) > 1 { + u, err := url.Parse(r.URL.String()) + if err != nil { + status := http.StatusInternalServerError + http.Error(w, "Failed to parse URL", status) + return status, -1, serve.Log + } + u.Path = u.Path[:len(u.Path)-1] + + status := http.StatusTemporaryRedirect + http.Redirect(w, r, u.String(), status) + return status, -1, serve.Log + } + + data, err := ioutil.ReadFile(filePath) + if err != nil { + status := http.StatusInternalServerError + http.Error(w, err.Error(), status) + return status, -1, serve.Log + } + + result := gmitohtml.Convert([]byte(data), r.URL.String()) + + status := http.StatusOK + w.Header().Set("Content-Type", htmlType) + w.WriteHeader(status) + + w.Write(result) + return status, int64(len(result)), serve.Log + } + } + } + + http.NotFound(w, r) + return http.StatusNotFound, -1, "" +} + +type responseWriter struct { + statusCode int + header http.Header + conn *tls.Conn + wroteHeader bool +} + +func newResponseWriter(conn *tls.Conn) *responseWriter { + return &responseWriter{ + header: http.Header{}, + conn: conn, + } +} + +func (w *responseWriter) Header() http.Header { + return w.header +} + +func (w *responseWriter) Write(b []byte) (int, error) { + if !w.wroteHeader { + w.wroteHeader = true + w.WriteHeader(http.StatusOK) + } + return w.conn.Write(b) +} + +func (w *responseWriter) WriteHeader(statusCode int) { + if w.wroteHeader { + return + } + + w.statusCode = statusCode + + statusText := http.StatusText(statusCode) + if statusText == "" { + statusText = "Unknown" + } + + w.conn.Write([]byte(fmt.Sprintf("HTTP/1.1 %d %s\r\n", statusCode, statusText))) + w.header.Write(w.conn) + w.conn.Write([]byte("\r\n")) +} diff --git a/server.go b/server.go index 91961f7..2d58738 100644 --- a/server.go +++ b/server.go @@ -6,8 +6,10 @@ import ( "crypto/tls" "crypto/x509" "fmt" + "io" "log" "net" + "net/http" "net/url" "os" "path" @@ -27,6 +29,7 @@ const ( plainType = "text/plain; charset=utf-8" geminiType = "text/gemini; charset=utf-8" htmlType = "text/html; charset=utf-8" + cssType = "text/css; charset=utf-8" logTimeFormat = "2006-01-02 15:04:05" ) @@ -303,13 +306,14 @@ func handleConn(c *tls.Conn) { var logPath string status := 0 size := int64(-1) + protocol := "Gemini" defer func() { if quiet && logPath == "" { return } - entry := logEntry(request, status, size, time.Since(t)) + entry := logEntry(protocol, request, status, size, time.Since(t)) if !quiet { log.Println(string(entry)) @@ -336,21 +340,33 @@ func handleConn(c *tls.Conn) { f.Write([]byte("\n")) }() - defer c.Close() c.SetReadDeadline(time.Now().Add(readTimeout)) + defer c.Close() - var requestData string - scanner := bufio.NewScanner(c) - if !config.SaneEOL { - scanner.Split(scanCRLF) + var dataBuf []byte + var buf = make([]byte, 1) + var readCR bool + if config.SaneEOL { + readCR = true } - if scanner.Scan() { - requestData = scanner.Text() - } - if err := scanner.Err(); err != nil { - status = writeStatus(c, statusBadRequest) - return + for { + n, err := c.Read(buf) + if err == io.EOF { + break + } else if err != nil || n != 1 { + return + } + + if buf[0] == '\r' { + readCR = true + continue + } else if readCR && buf[0] == '\n' { + break + } + + dataBuf = append(dataBuf, buf[0]) } + requestData := string(dataBuf) state := c.ConnectionState() certs := state.PeerCertificates @@ -363,6 +379,36 @@ func handleConn(c *tls.Conn) { clientCertKeys = append(clientCertKeys, pubKey) } + if strings.HasPrefix(requestData, "GET ") { + w := newResponseWriter(c) + defer w.WriteHeader(http.StatusOK) + + if config.DisableHTTPS { + status = statusProxyRequestRefused + http.Error(w, "Error: Proxy request refused", http.StatusBadRequest) + return + } + + r, err := http.ReadRequest(bufio.NewReader(io.MultiReader(strings.NewReader(requestData+"\r\n"), c))) + if err != nil { + status = http.StatusInternalServerError + http.Error(w, err.Error(), status) + return + } + if r.Proto == "" { + protocol = "HTTP" + } else { + protocol = r.Proto + } + r.URL.Scheme = "https" + r.URL.Host = strings.ToLower(r.Host) + + request = r.URL + + status, size, logPath = serveHTTPS(w, r) + return + } + if len(requestData) > urlMaxLength || !utf8.ValidString(requestData) { status = writeStatus(c, statusBadRequest) return @@ -374,6 +420,7 @@ func handleConn(c *tls.Conn) { status = writeStatus(c, statusBadRequest) return } + request.Host = strings.ToLower(request.Host) requestHostname := request.Hostname() if requestHostname == "" || strings.ContainsRune(requestHostname, ' ') { @@ -389,22 +436,16 @@ func handleConn(c *tls.Conn) { } } - if request.Scheme == "" { - request.Scheme = "gemini" - } - if request.Scheme != "gemini" || (requestPort > 0 && requestPort != config.port) { + validScheme := request.Scheme == "gemini" || (!config.DisableHTTPS && request.Scheme == "https") + if !validScheme || (requestPort > 0 && requestPort != config.port) { status = writeStatus(c, statusProxyRequestRefused) return } - if request.Scheme == "gemini" { - request.Host = strings.ToLower(request.Host) - } - status, size, logPath = handleRequest(c, request, requestData) } -func logEntry(request *url.URL, status int, size int64, elapsed time.Duration) []byte { +func logEntry(protocol string, request *url.URL, status int, size int64, elapsed time.Duration) []byte { hostFormatted := "-" pathFormatted := "-" sizeFormatted := "-" @@ -424,7 +465,7 @@ func logEntry(request *url.URL, status int, size int64, elapsed time.Duration) [ if size >= 0 { sizeFormatted = strconv.FormatInt(size, 10) } - return []byte(fmt.Sprintf(`%s - - - [%s] "GET %s Gemini" %d %s %.4f`, hostFormatted, time.Now().Format(logTimeFormat), pathFormatted, status, sizeFormatted, elapsed.Seconds())) + return []byte(fmt.Sprintf(`%s - - - [%s] "GET %s %s" %d %s %.4f`, hostFormatted, time.Now().Format(logTimeFormat), pathFormatted, protocol, status, sizeFormatted, elapsed.Seconds())) } func handleListener(l net.Listener) {