Establish multiple WebRTC connections

This commit is contained in:
Trevor Slocum 2019-12-09 16:16:24 -08:00
parent 03ce83fc34
commit 7f84951ccf
11 changed files with 475 additions and 189 deletions

5
HOSTING.md Normal file
View File

@ -0,0 +1,5 @@
This document covers how to host a harmony server.
# Requirements
- WIP

View File

@ -10,6 +10,23 @@ Voice and text communications platform
- Low-latency voice chat using [Opus](https://en.wikipedia.org/wiki/Opus_%28audio_format%29)
- Rich text chat using [Markdown](https://en.wikipedia.org/wiki/Markdown)
## Web client
The only client currently implemented is a web interface (located in pkg/web) hosted by the server.
## harmony-server
The server (located in cmd/harmony-server) passes voice and text communications between clients.
See [HOSTING.md](https://man.sr.ht/~tslocum/harmony/HOSTING.md) for more information on hosting a server.
## Support
Please share suggestions/issues [here](https://todo.sr.ht/~tslocum/harmony).
## Libraries
The following libraries are used to build netris:
* [webrtc](https://github.com/pion/webrtc) - WebRTC connections and audio
* [websocket](https://github.com/gorilla/websocket) - WebSocket connections

7
go.mod
View File

@ -4,15 +4,10 @@ go 1.13
require (
github.com/GeertJohan/go.rice v1.0.0
github.com/daaku/go.zipexe v1.0.1 // indirect
github.com/golang/protobuf v1.3.2 // indirect
github.com/gorilla/mux v1.7.3
github.com/gorilla/websocket v1.4.1
github.com/lucas-clemente/quic-go v0.13.1 // indirect
github.com/pion/ice v0.7.4 // indirect
github.com/pion/rtp v1.1.4
github.com/pion/webrtc/v2 v2.1.16
github.com/pkg/errors v0.8.1
golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e // indirect
golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933 // indirect
golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 // indirect
)

33
go.sum
View File

@ -2,13 +2,10 @@ github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo
github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ=
github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75 h1:3ILjVyslFbc4jl1w5TWuvvslFD/nDfR2H8tVaMVLrEY=
github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75/go.mod h1:uAXEEpARkRhCZfEvy/y0Jcc888f9tHCc1W7/UeEtreE=
github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
github.com/daaku/go.zipexe v1.0.0 h1:VSOgZtH418pH9L16hC/JrgSNJbbAL26pj7lmD1+CGdY=
github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E=
github.com/daaku/go.zipexe v1.0.1 h1:wV4zMsDOI2SZ2m7Tdz1Ps96Zrx+TzaK15VbUaGozw0M=
github.com/daaku/go.zipexe v1.0.1/go.mod h1:5xWogtqlYnfBXkSB1o9xysukNP9GTvaNkqzUZbt3Bw8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -16,11 +13,8 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk=
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
@ -33,15 +27,10 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9 h1:tbuodUh2vuhOVZAdW3NEUvosFHUMJwUNl7jk/VSEiwc=
github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw=
github.com/lucas-clemente/quic-go v0.13.1 h1:CxtJTXQIh2aboCPk0M6vf530XOov6DZjVBiSE3nSj8s=
github.com/lucas-clemente/quic-go v0.13.1/go.mod h1:Vn3/Fb0/77b02SGhQk36KzOUmXgVpFfizUfW5WMaqyU=
github.com/marten-seemann/chacha20 v0.2.0 h1:f40vqzzx+3GdOmzQoItkLX5WLvHgPgyYqFFIO5Gh4hQ=
github.com/marten-seemann/chacha20 v0.2.0/go.mod h1:HSdjFau7GzYRj+ahFNwsO3ouVJr1HFkWoEwNDb4TMtE=
github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA=
github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
github.com/marten-seemann/qtls v0.4.1 h1:YlT8QP3WCCvvok7MGEZkMldXbyqgr8oFg5/n8Gtbkks=
github.com/marten-seemann/qtls v0.4.1/go.mod h1:pxVXcHHw1pNIt8Qo0pwSYQEoZ8yYOOPXTCZLQQunvRc=
github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
@ -52,8 +41,9 @@ github.com/pion/datachannel v1.4.13 h1:ezTn3AtUtXvKemRRjRdUgao/T8bH4ZJwrpOqU8Iz3
github.com/pion/datachannel v1.4.13/go.mod h1:+rBUwEDonA63KXx994DP/ofyyGVAm6AIMvOqQZxjWRU=
github.com/pion/dtls/v2 v2.0.0-rc.3 h1:u9utI+EDJOjOWfrkGQsD8WNssPcTwfYIanFB6oI8K+4=
github.com/pion/dtls/v2 v2.0.0-rc.3/go.mod h1:x0XH+cN5z+l/+/4nYL8r4sB8g6+0d1Zp2Pfkcoz8BKY=
github.com/pion/ice v0.7.2 h1:b+QxnpJ7AVyFDXBOMnEypNXS+fZM8+4+itNInwrrI6U=
github.com/pion/ice v0.7.2/go.mod h1:xLKf+788DA/ZubtdBfiDT3vnEmIdiF5eDqjs4rzUAg8=
github.com/pion/ice v0.7.4 h1:cwbduOII1hRb7ntoDAyXvEqGxALBVFk7PO4IbC9i/9A=
github.com/pion/ice v0.7.4/go.mod h1:V+ILxpsY3qtzJnvg9W3mLgQ37FkYfzM4ri0wX3VLzpM=
github.com/pion/logging v0.2.1/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
@ -94,30 +84,25 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c h1:/nJuwDLoL/zrqY6gf57vxC+Pi+pZ8bfhpPkicO5H7W4=
golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e h1:egKlR8l7Nu9vHGWbcUV8lqR4987UfUbBd7GbhqGzNYU=
golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933 h1:e6HwijUxhDe+hPNjZQQn9bA5PW3vNmnN64U2ZW759Lk=
golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c h1:S/FtSvpNLtFBgjTqcKsRpsa6aVsI6iztaz1bQd9BJwE=
golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 h1:ZBzSG/7F4eNKz2L3GE9o300RX0Az1Bw5HF7PDraD+qU=
golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

7
pkg/audio/audio.go Normal file
View File

@ -0,0 +1,7 @@
package audio
const (
ClockRate = 48 // KHz
FrameTime = 20 // ms
Samples = ClockRate * FrameTime
)

View File

@ -4,30 +4,44 @@ import (
"encoding/json"
"log"
"strconv"
"sync"
"time"
"github.com/gorilla/websocket"
"github.com/pion/rtp"
"github.com/pion/webrtc/v2"
"github.com/pkg/errors"
)
type Client struct {
ID int
Name string
Status int
Conn *websocket.Conn
PeerConn *webrtc.PeerConnection
In chan *Message
Out chan *Message
ID int
Name string
Status int
Conn *websocket.Conn
AudioTrack *webrtc.Track
PeerConns map[int]*webrtc.PeerConnection
PeerConnLock *sync.Mutex
In chan *Message
Out chan *Message
AudioTracks map[int]*webrtc.Track
SequenceNumber uint16
VoiceIn chan []int16
VoiceOut []chan *rtp.Packet
VoiceOutClient []int
VoiceOutActive []time.Time
VoiceOutLock *sync.Mutex
Terminated chan bool
}
func NewClient(conn *websocket.Conn) *Client {
c := Client{Conn: conn, Name: "Anonymous", In: make(chan *Message, 10), Out: make(chan *Message, 10), SequenceNumber: 1, Terminated: make(chan bool)}
c := Client{Conn: conn, Name: "Anonymous", PeerConns: make(map[int]*webrtc.PeerConnection), PeerConnLock: new(sync.Mutex), In: make(chan *Message, 10), Out: make(chan *Message, 10), SequenceNumber: 1, AudioTracks: make(map[int]*webrtc.Track), VoiceIn: make(chan []int16, 10), Terminated: make(chan bool)}
c.VoiceOutLock = new(sync.Mutex)
go c.handleRead()
go c.handleWrite()
@ -98,11 +112,9 @@ func (c *Client) Close() {
return
}
c.Status = -1
c.AudioTrack = nil
if c.PeerConn != nil {
c.PeerConn.Close()
}
c.CloseAudio()
if c.Conn != nil {
c.Conn.Close()
}
@ -118,13 +130,73 @@ func (c *Client) Close() {
}()
}
func (c *Client) ClosePeerConn() {
if c.PeerConn == nil {
func (c *Client) InitAudio() {
c.VoiceOutLock.Lock()
defer c.VoiceOutLock.Unlock()
if len(c.VoiceOut) > 0 {
return
}
c.PeerConn.Close()
c.AudioTrack = nil
c.PeerConn = nil
c.SequenceNumber = 1
for i := 0; i < 3; i++ {
c.VoiceOut = append(c.VoiceOut, make(chan *rtp.Packet, 10))
c.VoiceOutClient = append(c.VoiceOutClient, 0)
c.VoiceOutActive = append(c.VoiceOutActive, time.Time{})
}
}
func (c *Client) WriteAudio(p *rtp.Packet, source int) {
c.VoiceOutLock.Lock()
for i := range c.VoiceOut {
if c.VoiceOutClient[i] == 0 || c.VoiceOutClient[i] == source || time.Since(c.VoiceOutActive[i]) >= 50*time.Millisecond {
select {
case c.VoiceOut[i] <- p:
default:
log.Printf("client %d warning: filled voice out buffer when writing from %d", c.ID, source)
continue
}
c.VoiceOutActive[i] = time.Now()
c.VoiceOutClient[i] = source
c.VoiceOutLock.Unlock()
return
}
}
c.VoiceOutLock.Unlock()
}
func (c *Client) CloseAudio() {
c.ClosePeerConns()
c.PeerConns = make(map[int]*webrtc.PeerConnection)
c.VoiceOut = nil
c.VoiceOutClient = nil
c.VoiceOutActive = nil
}
func (c *Client) ClosePeerConns() {
for id := range c.PeerConns {
c.ClosePeerConn(id)
}
}
func (c *Client) ClosePeerConn(id int) {
c.VoiceOutLock.Lock()
defer c.VoiceOutLock.Unlock()
pc := c.PeerConns[id]
if pc == nil {
return
}
select {
case c.VoiceOut[id] <- nil:
default:
log.Println("failed to close channel for client " + strconv.Itoa(c.ID))
}
pc.Close()
pc = nil
c.AudioTracks[id] = nil
}

View File

@ -15,6 +15,7 @@ const (
MessageAction MessageType = 115
MessageDisconnect MessageType = 119
MessageChat MessageType = 120
MessageUsers MessageType = 121
)
func (t MessageType) String() string {

View File

@ -1,3 +1,5 @@
var printStats = false;
var socket = null;
var ReconnectDelay = 0;
var reconnectTimeout;
@ -6,14 +8,15 @@ var connected;
var chatprefix = "";
var voice = false;
var ptt = false;
var printStats = false;
var nickname = "Anonymous";
var pc;
var numConnections = 3;
var peerConnections = [];
var RTCConstraints = {
audio: {
autoGainControl: true,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: false,
echoCancellation: false,
noiseSuppression: false,
},
video: false
};
@ -28,6 +31,8 @@ var audioTrack;
var shownPTTHelp = false;
var muteOnMouseUp = true;
var userListStatus = '';
var MessageBinary = 2;
var MessagePing = 100;
var MessageCall = 101;
@ -40,6 +45,7 @@ var MessageTopic = 114;
var MessageAction = 115;
var MessageDisconnect = 119;
var MessageChat = 120;
var MessageUsers = 121;
var tagsToReplace = {
'&': '&amp;',
@ -75,55 +81,14 @@ function HandleInput(e) {
$(document).ready(function () {
$("#voiceButtonJoin").on("click touchstart", function () {
pc = new RTCPeerConnection({
iceServers: RTCICEServers
});
pc.onicecandidate = event => {
if (event.candidate === null) {
if (peerConnections.length > 0 || voice) {
return;
}
}
};
navigator.mediaDevices.getUserMedia(RTCConstraints).then(localStream => {
audioTracks = localStream.getAudioTracks();
if (audioTracks.length > 0) {
console.log(`Using Audio device: ${audioTracks[0].label} - tracks available: ${audioTracks}`);
}
audioTrack = audioTracks[0];
pc.addTrack(audioTrack);
pc.getSenders()[0].replaceTrack(null);
pc.ontrack = function (event) {
console.log(`onTrack ${event.streams.length} ${event.streams[0].getAudioTracks().length}`);
pc.addTransceiver(event.streams[0].getAudioTracks()[0], {'direction': 'sendrecv'});
var el = document.getElementById('audioplayer');
el.autoplay = true;
el.srcObject = event.streams[0];
voice = true;
if (!shownPTTHelp) {
shownPTTHelp = true;
Log("* Note: Push-to-talk is bound to &lt;F8&gt;");
}
$('#voicepttcontainer').css('display', 'table-row');
$('#voiceinactiveleft').css('display', 'none');
$('#voiceactiveleft').css('display', 'inline-block');
$('#voiceinactiveright').css('display', 'none');
$('#voiceactiveright').css('display', 'inline-block');
$('#voiceButton').html('Quit voice chat');
updateVoiceStatus();
};
pc.createOffer(RTCOfferOptions)
.then(onRTCDescription, onRTCDescriptionError);
}).catch(Log);
var i;
for (i = 0; i < numConnections; i++) {
peerConnections.push(createPeerConnection(i));
}
});
$("#voiceptt").on("touchstart", function (e) {
@ -165,10 +130,18 @@ $(document).ready(function () {
});
$("#voiceButtonQuit").on("click touchstart", function () {
pc.close();
w(MessageQuit, "");
if (!voice) {
return;
}
voice = false;
w(MessageQuit, "");
peerConnections.forEach(function (pc) {
pc.close();
});
peerConnections = [];
$('#voicepttcontainer').css('display', 'none');
$('#voiceinactiveleft').css('display', 'inline-block');
$('#voiceactiveleft').css('display', 'none');
@ -179,8 +152,6 @@ $(document).ready(function () {
updateVoiceStatus();
});
Connect();
window.setInterval(() => {
if (!webSocketReady()) {
return;
@ -191,6 +162,8 @@ $(document).ready(function () {
if (printStats) {
window.setInterval(() => {
// TODO Fix
if (!pc) {
return;
}
@ -223,24 +196,103 @@ $(document).ready(function () {
});
}, 1000);
}
nickname = prompt("What is your name?", nickname);
Connect();
});
function onRTCDescriptionError(error) {
console.log(`Failed to create/set session description: ${error.toString()}`);
function createPeerConnection(id) {
var pc = new RTCPeerConnection({
iceServers: RTCICEServers
});
pc.onicecandidate = event => {
if (event.candidate === null) {
}
};
if (id > 0) {
pc.addTransceiver('audio', {'direction': 'recvonly'});
}
pc.ontrack = function (event) {
console.log(`PC ${id} onTrack ${event.streams.length} ${event.streams[0].id} ${event.streams[0].getAudioTracks().length}`);
if (id > 0) {
pc.addTransceiver(event.streams[0].getAudioTracks()[0], {'direction': 'sendrecv'});
}
$("#hidden").append('<audio id="audiostream' + id + '"></audio>');
var el = document.getElementById('audiostream' + id);
el.autoplay = true;
el.srcObject = event.streams[0];
voice = true;
if (id == 0 && !shownPTTHelp) {
shownPTTHelp = true;
Log("* Note: Push-to-talk is bound to &lt;F8&gt;");
}
$('#voicepttcontainer').css('display', 'table-row');
$('#voiceinactiveleft').css('display', 'none');
$('#voiceactiveleft').css('display', 'inline-block');
$('#voiceinactiveright').css('display', 'none');
$('#voiceactiveright').css('display', 'inline-block');
$('#voiceButton').html('Quit voice chat');
updateVoiceStatus();
};
if (id == 0) {
navigator.mediaDevices.getUserMedia(RTCConstraints).then(localStream => {
audioTracks = localStream.getAudioTracks();
if (audioTracks.length > 0) {
console.log(`PC ${id} Using Audio device: ${audioTracks[0].label} - tracks available: ${audioTracks}`);
}
audioTrack = audioTracks[0];
pc.addTrack(audioTrack);
pc.getSenders()[0].replaceTrack(null);
pc.createOffer(RTCOfferOptions)
.then(onRTCDescription(id), onRTCDescriptionError(id));
}).catch(Log);
} else {
pc.createOffer(RTCOfferOptions)
.then(onRTCDescription(id), onRTCDescriptionError(id));
}
return pc;
}
function onRTCDescription(desc) {
console.log(`Offer from pc\n${desc.sdp}`);
pc.setLocalDescription(desc)
.then(() => {
desc.sdp = maybePreferCodec(desc.sdp, 'audio', 'send', 'opus');
desc.sdp = maybePreferCodec(desc.sdp, 'audio', 'receive', 'opus');
function onRTCDescriptionError(id) {
return function (error) {
console.log(`Failed to create/set session description: ${error.toString()}`);
}
}
console.log("onRTCDescription");
console.log(desc);
function onRTCDescription(id) {
return function (desc) {
console.log(`PC ${id} Offer received \n${desc.sdp}`);
w(MessageCall, desc.sdp);
}, onRTCDescriptionError);
if (peerConnections.length < id) {
return;
}
peerConnections[id].setLocalDescription(desc)
.then(() => {
desc.sdp = maybePreferCodec(desc.sdp, 'audio', 'send', 'opus');
desc.sdp = maybePreferCodec(desc.sdp, 'audio', 'receive', 'opus');
console.log(`PC ${id} onRTCDescription`);
wpc(id, MessageCall, desc.sdp);
}, onRTCDescriptionError);
}
}
function Connect() {
@ -275,6 +327,8 @@ function Connect() {
clearTimeout(reconnectTimeout);
}
w(MessageNick, nickname);
updateVoiceStatus();
};
socket.onmessage = function (e) {
@ -290,17 +344,59 @@ function Connect() {
}
if (p.T == MessageAnswer) {
if (p.PC === undefined || p.PC > peerConnections.length) {
return;
}
var pc = peerConnections[p.PC];
pc.setRemoteDescription(new RTCSessionDescription({type: 'answer', sdp: p.M}));
} else if (p.T == MessageConnect) {
Log("* " + p.M + " connected");
if (p.N === undefined) {
return;
}
Log("* " + escapeEntities(p.N) + " connected");
} else if (p.T == MessageJoin) {
Log("* " + p.M + " joined #lobby voice chat");
if (p.N === undefined) {
return;
}
Log("* " + escapeEntities(p.N) + " joined #lobby voice chat");
} else if (p.T == MessageQuit) {
Log("* " + p.M + " quit #lobby voice chat");
if (p.N === undefined) {
return;
}
Log("* " + escapeEntities(p.N) + " quit #lobby voice chat");
} else if (p.T == MessageDisconnect) {
Log("* " + p.M + " disconnected");
if (p.N === undefined) {
return;
}
Log("* " + escapeEntities(p.N) + " disconnected");
} else if (p.T == MessageChat) {
Log("&lt;Anonymous&gt; " + escapeEntities(p.M));
if (p.N === undefined) {
return;
}
Log("&lt;" + escapeEntities(p.N) + "&gt; " + escapeEntities(p.M));
} else if (p.T == MessageUsers) {
var usersconnected = 0;
var usersvoice = 0;
var u = JSON.parse(p.M);
for (let i = 0; i < u.length; i++) {
usersconnected++;
if (u[i].V) {
usersvoice++;
}
}
userListStatus = "Users: " + usersconnected + " - Voice chatting: " + usersvoice;
$("#voiceinactiveright").html(userListStatus);
updateVoiceStatus();
}
} else {
// TODO Binary data
@ -354,26 +450,26 @@ function Log(msg) {
}
function StartPTT() {
if (ptt) {
if (ptt || !voice || peerConnections.length == 0) {
return;
}
ptt = true;
var sender = pc.getSenders()[0];
var sender = peerConnections[0].getSenders()[0];
sender.replaceTrack(audioTrack);
updateVoiceStatus();
}
function StopPTT() {
if (!ptt) {
if (!ptt || !voice || peerConnections.length == 0) {
return;
}
ptt = false;
var sender = pc.getSenders()[0];
var sender = peerConnections[0].getSenders()[0];
sender.replaceTrack(null);
updateVoiceStatus();
@ -383,7 +479,7 @@ function updateVoiceStatus() {
if (ptt) {
$('#voiceactiveleft').html('<b>Transmitting</b>');
} else {
$('#voiceactiveleft').html('# users voice chatting');
$('#voiceactiveleft').html(userListStatus);
}
}
@ -395,6 +491,14 @@ function w(t, m) {
socket.send(JSON.stringify({T: t, M: btoa(m)}));
}
function wpc(pc, t, m) {
if (!webSocketReady()) {
return;
}
socket.send(JSON.stringify({PC: pc, T: t, M: btoa(m)}));
}
function escapeEntitiesCallback(tag) {
return tagsToReplace[tag] || tag;
}

View File

@ -11,6 +11,7 @@
<tr>
<td id="chatcontainer" colspan="2">
<div id="chathistory"></div>
<div id="stats"></div>
</td>
</tr>
<tr id="inputheader">
@ -36,11 +37,11 @@
<div id="voiceinactiveleft">
<button id="voiceButtonJoin">Join voice chat</button>
</div>
<div id="voiceactiveleft" style="display: none;"># users voice chatting</div>
<div id="voiceactiveleft" style="display: none;"></div>
</td>
<td align="right">
<div id="voiceinactiveright" style="">
# users voice chatting
&nbsp;
</div>
<div id="voiceactiveright" style="display: none;">
<button id="voiceButtonQuit">Quit voice chat</button>
@ -48,9 +49,7 @@
</td>
</tr>
</table>
<div id="stats" style="display: none"></div>
<div style="display: none">
<audio id="audioplayer"></audio>
<div style="display: none" id="hidden">
</div>
</body>
</html>

21
pkg/web/user.go Normal file
View File

@ -0,0 +1,21 @@
package web
import "regexp"
type User struct {
N string
V bool
}
var nickRegexp = regexp.MustCompile(`[^a-zA-Z0-9_\-!@#$%^&*+=,./?]+`)
func Nickname(nick string) string {
nick = nickRegexp.ReplaceAllString(nick, "")
if len(nick) > 10 {
nick = nick[:10]
} else if nick == "" {
nick = "Anonymous"
}
return nick
}

View File

@ -1,17 +1,22 @@
package web
import "C"
import (
"io"
"encoding/json"
"log"
"math/rand"
"net/http"
"strconv"
"strings"
"sync"
"time"
"git.sr.ht/~tslocum/harmony/pkg/audio"
rice "github.com/GeertJohan/go.rice"
"github.com/gorilla/mux"
"github.com/gorilla/websocket"
"github.com/pion/rtp"
"github.com/pion/webrtc/v2"
"github.com/pion/webrtc/v2/pkg/media"
"github.com/pkg/errors"
)
@ -35,9 +40,11 @@ var upgrader = websocket.Upgrader{
}
type Message struct {
S int // Source
T MessageType // Type
M []byte // Message
S int // Source
N string // Source nickname
PC int // PeerConn
T MessageType // Type
M []byte // Message
}
type WebInterface struct {
@ -56,25 +63,40 @@ func NewWebInterface(address string, path string) *WebInterface {
go w.handleIncomingClients()
go func() {
if err := http.ListenAndServe(address, r); err != nil {
log.Fatal("Web server error: ", err)
}
}()
addressSplit := strings.Split(address, ",")
for _, add := range addressSplit {
add := add // Capture
go func() {
if err := http.ListenAndServe(add, r); err != nil {
log.Fatal("failed to listen on address ", add, ":", err)
}
}()
}
return &w
}
func (w *WebInterface) handleIncomingClients() {
for c := range incomingClients {
c := c // Capture
w.ClientsLock.Lock()
id := w.nextClientID()
c.ID = id
w.Clients[id] = c
for _, wc := range w.Clients {
wc.Out <- &Message{T: MessageConnect, M: []byte(c.Name)}
}
go func(c *Client) {
time.Sleep(500 * time.Millisecond)
w.updateUserList()
w.ClientsLock.Lock()
for _, wc := range w.Clients {
wc.Out <- &Message{T: MessageConnect, N: c.Name, M: []byte(c.Name)}
}
w.ClientsLock.Unlock()
}(c)
w.ClientsLock.Unlock()
@ -99,44 +121,52 @@ func (w *WebInterface) handleRead(c *Client) {
case MessagePing:
// TODO Handle ping
case MessageCall:
answer, err := w.answerRTC(c, msg.M)
answer, err := w.answerRTC(c, msg.PC, msg.M)
if err != nil {
log.Printf("failed to answer call: %s", err)
continue
}
c.Out <- &Message{T: MessageAnswer, M: answer}
c.Out <- &Message{T: MessageAnswer, PC: msg.PC, M: answer}
case MessageChat:
w.ClientsLock.Lock()
for _, wc := range w.Clients {
wc.Out <- &Message{S: c.ID, T: MessageChat, M: []byte(msg.M)}
wc.Out <- &Message{S: c.ID, N: c.Name, T: MessageChat, M: []byte(msg.M)}
}
w.ClientsLock.Unlock()
log.Printf("<%s> %s", c.Name, msg.M)
case MessageConnect, MessageJoin, MessageNick, MessageQuit, MessageDisconnect:
case MessageNick:
w.ClientsLock.Lock()
oldNick := c.Name
c.Name = Nickname(string(msg.M))
if oldNick != "Anonymous" {
msg := &Message{S: c.ID, N: oldNick, T: MessageNick, M: []byte(c.Name)}
for _, wc := range w.Clients {
wc.Out <- msg
}
}
w.ClientsLock.Unlock()
case MessageConnect, MessageJoin, MessageQuit, MessageDisconnect:
w.ClientsLock.Lock()
if msg.T == MessageQuit || msg.T == MessageDisconnect {
c.AudioTrack = nil
if c.PeerConn != nil {
c.ClosePeerConn()
log.Printf("closing peerconn, peer sent %s %d", msg.T, c.ID)
}
c.CloseAudio()
if msg.T == MessageDisconnect {
c.Close()
}
}
msg.M = []byte(c.Name)
msg.N = c.Name
for _, wc := range w.Clients {
if (msg.T == MessageJoin || msg.T == MessageQuit) && wc.AudioTrack == nil && wc.ID != c.ID {
if (msg.T == MessageJoin || msg.T == MessageQuit) && len(wc.AudioTracks) == 0 && wc.ID != c.ID {
continue
}
@ -144,6 +174,8 @@ func (w *WebInterface) handleRead(c *Client) {
}
w.ClientsLock.Unlock()
w.updateUserList()
default:
log.Printf("Unhandled message %d %s", msg.T, msg.M)
}
@ -184,15 +216,44 @@ func (w *WebInterface) webSocketHandler(wr http.ResponseWriter, r *http.Request)
delete(w.Clients, id)
for _, wc := range w.Clients {
wc.Out <- &Message{T: MessageDisconnect, M: []byte(name)}
wc.Out <- &Message{T: MessageDisconnect, N: name, M: []byte(name)}
}
}
w.ClientsLock.Unlock()
}
func (w *WebInterface) answerRTC(c *Client, sdp []byte) ([]byte, error) {
if c.PeerConn != nil {
return nil, errors.New("already have peerconn")
func (w *WebInterface) updateUserList() {
w.ClientsLock.Lock()
msg := &Message{T: MessageUsers}
var userList []*User
for _, wc := range w.Clients {
userList = append(userList, &User{N: wc.Name, V: len(wc.VoiceOut) > 0})
}
var err error
msg.M, err = json.Marshal(userList)
if err != nil {
log.Fatal("failed to marshal user list: ", err)
}
for _, wc := range w.Clients {
wc.Out <- msg
}
w.ClientsLock.Unlock()
}
func (w *WebInterface) answerRTC(c *Client, peerConnID int, sdp []byte) ([]byte, error) {
c.PeerConnLock.Lock()
defer c.PeerConnLock.Unlock()
/*if c.NextPeerConn >= 3 {
return nil, errors.New("already have 3 peerconns") // TODO configurable
}*/
if c.PeerConns[peerConnID] != nil {
return nil, errors.New("already have next peerconn")
}
offer := webrtc.SessionDescription{Type: webrtc.SDPTypeOffer, SDP: string(sdp)}
@ -216,7 +277,11 @@ func (w *WebInterface) answerRTC(c *Client, sdp []byte) ([]byte, error) {
panic(err)
}
c.PeerConn = pc
if len(c.PeerConns) == 0 {
c.InitAudio()
}
c.PeerConns[peerConnID] = pc
err = pc.SetRemoteDescription(offer)
if err != nil {
@ -231,51 +296,60 @@ func (w *WebInterface) answerRTC(c *Client, sdp []byte) ([]byte, error) {
}
}
if payloadType == 0 {
c.ClosePeerConn()
c.ClosePeerConn(peerConnID)
return nil, errors.New("no payloadType")
}
t, err := pc.NewTrack(payloadType, rand.Uint32(), "audio", "harmony")
name := "harmony-audio-" + strconv.Itoa(peerConnID)
c.AudioTracks[peerConnID], err = pc.NewTrack(payloadType, 1000+uint32(peerConnID), name, name)
if err != nil {
panic(err)
}
c.AudioTrack = t
_, err = pc.AddTrack(c.AudioTrack)
direction := webrtc.RTPTransceiverDirectionSendonly
if peerConnID == 0 {
direction = webrtc.RTPTransceiverDirectionSendrecv
}
_, err = pc.AddTransceiverFromTrack(c.AudioTracks[peerConnID], webrtc.RtpTransceiverInit{Direction: direction})
if err != nil {
panic(err)
}
go func() {
for p := range c.VoiceOut[peerConnID] {
if p == nil {
return
}
err = c.AudioTracks[peerConnID].WriteSample(media.Sample{Data: p.Payload, Samples: audio.Samples})
if err != nil {
panic(err)
}
}
}()
pc.OnTrack(func(remoteTrack *webrtc.Track, receiver *webrtc.RTPReceiver) {
var (
p *rtp.Packet
err error
)
log.Printf("client %d ontrack", c.ID)
var p *rtp.Packet
for {
p, err = remoteTrack.ReadRTP()
if err != nil {
c.ClosePeerConn()
c.ClosePeerConn(peerConnID)
log.Printf("failed to read RTP from %d", c.ID)
return
}
// TODO trim initial x ms transmitting to remove noise (configurable)
w.ClientsLock.Lock()
for _, wc := range w.Clients {
if wc.ID == c.ID || wc.AudioTrack == nil {
for ci, wc := range w.Clients {
if ci == c.ID {
continue
}
p.SSRC = wc.AudioTrack.SSRC()
p.PayloadType = wc.AudioTrack.PayloadType()
p.SequenceNumber = wc.SequenceNumber
if err = wc.AudioTrack.WriteRTP(p); err != nil && err != io.ErrClosedPipe {
panic(err)
}
wc.SequenceNumber++
wc.WriteAudio(p, c.ID)
}
w.ClientsLock.Unlock()
}
@ -284,28 +358,34 @@ func (w *WebInterface) answerRTC(c *Client, sdp []byte) ([]byte, error) {
pc.OnConnectionStateChange(func(connectionState webrtc.PeerConnectionState) {
log.Printf("%d conn state -> %s\n", c.ID, connectionState)
if peerConnID != 0 {
return // Process events from first PeerConn only
}
if connectionState == webrtc.PeerConnectionStateConnected || connectionState == webrtc.PeerConnectionStateDisconnected {
w.ClientsLock.Lock()
if connectionState == webrtc.PeerConnectionStateDisconnected {
c.ClosePeerConn()
c.ClosePeerConn(peerConnID)
log.Printf("closing peerconn, peer disconnected %d", c.ID)
}
for _, wc := range w.Clients {
if wc.AudioTrack == nil && wc.ID != c.ID {
if len(wc.AudioTracks) == 0 && wc.ID != c.ID {
continue
}
if connectionState == webrtc.PeerConnectionStateConnected {
wc.Out <- &Message{T: MessageJoin, M: []byte(c.Name)}
wc.Out <- &Message{T: MessageJoin, N: c.Name, M: []byte(c.Name)}
} else {
wc.Out <- &Message{T: MessageQuit, M: []byte(c.Name)}
wc.Out <- &Message{T: MessageQuit, N: c.Name, M: []byte(c.Name)}
}
}
w.ClientsLock.Unlock()
w.updateUserList()
}
})