From 177e818d5298702f685e0bd0e45093d5b79b33c9 Mon Sep 17 00:00:00 2001 From: Trevor Slocum Date: Fri, 15 Sep 2017 00:27:30 -0700 Subject: [PATCH] Initial commit --- .gitignore | 4 + .gitlab-ci.yml | 24 + Gopkg.lock | 69 + Gopkg.toml | 38 + LICENSE | 674 + README.md | 6 + sage.default.conf | 5 + sage.go | 377 + vendor/github.com/BurntSushi/toml/.gitignore | 5 + vendor/github.com/BurntSushi/toml/.travis.yml | 15 + vendor/github.com/BurntSushi/toml/COMPATIBLE | 3 + vendor/github.com/BurntSushi/toml/COPYING | 21 + vendor/github.com/BurntSushi/toml/Makefile | 19 + vendor/github.com/BurntSushi/toml/README.md | 218 + .../BurntSushi/toml/_examples/example.go | 61 + .../BurntSushi/toml/_examples/example.toml | 35 + .../BurntSushi/toml/_examples/hard.toml | 22 + .../BurntSushi/toml/_examples/implicit.toml | 4 + .../toml/_examples/invalid-apples.toml | 6 + .../BurntSushi/toml/_examples/invalid.toml | 35 + .../BurntSushi/toml/_examples/readme1.toml | 5 + .../BurntSushi/toml/_examples/readme2.toml | 1 + .../toml/cmd/toml-test-decoder/COPYING | 14 + .../toml/cmd/toml-test-decoder/README.md | 13 + .../toml/cmd/toml-test-decoder/main.go | 90 + .../toml/cmd/toml-test-encoder/COPYING | 14 + .../toml/cmd/toml-test-encoder/README.md | 13 + .../toml/cmd/toml-test-encoder/main.go | 131 + .../BurntSushi/toml/cmd/tomlv/COPYING | 14 + .../BurntSushi/toml/cmd/tomlv/README.md | 21 + .../BurntSushi/toml/cmd/tomlv/main.go | 61 + vendor/github.com/BurntSushi/toml/decode.go | 509 + .../github.com/BurntSushi/toml/decode_meta.go | 121 + .../github.com/BurntSushi/toml/decode_test.go | 1461 ++ vendor/github.com/BurntSushi/toml/doc.go | 27 + vendor/github.com/BurntSushi/toml/encode.go | 568 + .../github.com/BurntSushi/toml/encode_test.go | 615 + .../BurntSushi/toml/encoding_types.go | 19 + .../BurntSushi/toml/encoding_types_1.1.go | 18 + vendor/github.com/BurntSushi/toml/lex.go | 953 + vendor/github.com/BurntSushi/toml/parse.go | 592 + vendor/github.com/BurntSushi/toml/session.vim | 1 + .../github.com/BurntSushi/toml/type_check.go | 91 + .../github.com/BurntSushi/toml/type_fields.go | 242 + vendor/github.com/coreos/bbolt/.gitignore | 4 + vendor/github.com/coreos/bbolt/LICENSE | 20 + vendor/github.com/coreos/bbolt/Makefile | 18 + vendor/github.com/coreos/bbolt/README.md | 852 + vendor/github.com/coreos/bbolt/appveyor.yml | 18 + vendor/github.com/coreos/bbolt/bolt_386.go | 7 + vendor/github.com/coreos/bbolt/bolt_amd64.go | 7 + vendor/github.com/coreos/bbolt/bolt_arm.go | 7 + vendor/github.com/coreos/bbolt/bolt_arm64.go | 9 + vendor/github.com/coreos/bbolt/bolt_linux.go | 10 + .../github.com/coreos/bbolt/bolt_openbsd.go | 27 + vendor/github.com/coreos/bbolt/bolt_ppc.go | 9 + vendor/github.com/coreos/bbolt/bolt_ppc64.go | 9 + .../github.com/coreos/bbolt/bolt_ppc64le.go | 9 + vendor/github.com/coreos/bbolt/bolt_s390x.go | 9 + vendor/github.com/coreos/bbolt/bolt_unix.go | 89 + .../coreos/bbolt/bolt_unix_solaris.go | 90 + .../github.com/coreos/bbolt/bolt_windows.go | 144 + .../github.com/coreos/bbolt/boltsync_unix.go | 8 + vendor/github.com/coreos/bbolt/bucket.go | 748 + vendor/github.com/coreos/bbolt/bucket_test.go | 1867 ++ .../github.com/coreos/bbolt/cmd/bolt/main.go | 1532 ++ .../coreos/bbolt/cmd/bolt/main_test.go | 185 + vendor/github.com/coreos/bbolt/cursor.go | 400 + vendor/github.com/coreos/bbolt/cursor_test.go | 817 + vendor/github.com/coreos/bbolt/db.go | 1036 + vendor/github.com/coreos/bbolt/db_test.go | 1706 ++ vendor/github.com/coreos/bbolt/doc.go | 44 + vendor/github.com/coreos/bbolt/errors.go | 71 + vendor/github.com/coreos/bbolt/freelist.go | 248 + .../github.com/coreos/bbolt/freelist_test.go | 158 + vendor/github.com/coreos/bbolt/node.go | 604 + vendor/github.com/coreos/bbolt/node_test.go | 156 + vendor/github.com/coreos/bbolt/page.go | 178 + vendor/github.com/coreos/bbolt/page_test.go | 72 + vendor/github.com/coreos/bbolt/quick_test.go | 79 + .../coreos/bbolt/simulation_test.go | 329 + vendor/github.com/coreos/bbolt/tx.go | 682 + vendor/github.com/coreos/bbolt/tx_test.go | 716 + .../github.com/dustin/go-humanize/.gitignore | 6 + .../github.com/dustin/go-humanize/.travis.yml | 21 + vendor/github.com/dustin/go-humanize/LICENSE | 21 + .../dustin/go-humanize/README.markdown | 92 + vendor/github.com/dustin/go-humanize/big.go | 31 + .../github.com/dustin/go-humanize/bigbytes.go | 173 + .../dustin/go-humanize/bigbytes_test.go | 220 + vendor/github.com/dustin/go-humanize/bytes.go | 143 + .../dustin/go-humanize/bytes_test.go | 146 + vendor/github.com/dustin/go-humanize/comma.go | 108 + .../dustin/go-humanize/comma_test.go | 136 + .../github.com/dustin/go-humanize/commaf.go | 40 + .../dustin/go-humanize/commaf_test.go | 44 + .../dustin/go-humanize/common_test.go | 18 + vendor/github.com/dustin/go-humanize/ftoa.go | 23 + .../dustin/go-humanize/ftoa_test.go | 55 + .../github.com/dustin/go-humanize/humanize.go | 8 + .../github.com/dustin/go-humanize/number.go | 192 + .../dustin/go-humanize/number_test.go | 79 + .../github.com/dustin/go-humanize/ordinals.go | 25 + .../dustin/go-humanize/ordinals_test.go | 22 + vendor/github.com/dustin/go-humanize/si.go | 113 + .../github.com/dustin/go-humanize/si_test.go | 101 + vendor/github.com/dustin/go-humanize/times.go | 117 + .../dustin/go-humanize/times_test.go | 113 + vendor/github.com/fluffle/goirc/.gitignore | 8 + vendor/github.com/fluffle/goirc/.travis.yml | 19 + vendor/github.com/fluffle/goirc/LICENSE | 27 + vendor/github.com/fluffle/goirc/README.md | 119 + vendor/github.com/fluffle/goirc/client.go | 104 + .../fluffle/goirc/client/commands.go | 304 + .../fluffle/goirc/client/commands_test.go | 205 + .../fluffle/goirc/client/connection.go | 581 + .../fluffle/goirc/client/connection_test.go | 585 + .../fluffle/goirc/client/dispatch.go | 202 + .../fluffle/goirc/client/dispatch_test.go | 201 + vendor/github.com/fluffle/goirc/client/doc.go | 34 + .../fluffle/goirc/client/handlers.go | 105 + .../fluffle/goirc/client/handlers_test.go | 451 + .../github.com/fluffle/goirc/client/line.go | 216 + .../fluffle/goirc/client/line_test.go | 186 + .../fluffle/goirc/client/mocknetconn_test.go | 154 + .../fluffle/goirc/client/state_handlers.go | 262 + .../github.com/fluffle/goirc/doc/rfc2811.txt | 1067 + .../github.com/fluffle/goirc/doc/rfc2812.txt | 3531 ++++ .../fluffle/goirc/doc/unreal32docs.html | 3666 ++++ .../fluffle/goirc/logging/glog/glog.go | 32 + .../fluffle/goirc/logging/golog/golog.go | 15 + .../fluffle/goirc/logging/logging.go | 43 + .../github.com/fluffle/goirc/state/channel.go | 350 + .../fluffle/goirc/state/channel_test.go | 176 + .../fluffle/goirc/state/mock_tracker.go | 201 + vendor/github.com/fluffle/goirc/state/nick.go | 200 + .../fluffle/goirc/state/nick_test.go | 88 + .../github.com/fluffle/goirc/state/tracker.go | 366 + .../fluffle/goirc/state/tracker_test.go | 564 + vendor/github.com/fluffle/goirc/vims | 1 + vendor/github.com/golang/mock/.gitignore | 17 + vendor/github.com/golang/mock/.travis.yml | 13 + vendor/github.com/golang/mock/AUTHORS | 12 + vendor/github.com/golang/mock/CONTRIBUTORS | 37 + vendor/github.com/golang/mock/LICENSE | 202 + vendor/github.com/golang/mock/README.md | 86 + vendor/github.com/golang/mock/gomock/call.go | 258 + .../golang/mock/gomock/call_test.go | 47 + .../github.com/golang/mock/gomock/callset.go | 76 + .../golang/mock/gomock/controller.go | 183 + .../golang/mock/gomock/controller_test.go | 475 + .../github.com/golang/mock/gomock/matchers.go | 99 + .../golang/mock/gomock/matchers_test.go | 70 + .../mock/gomock/mock_matcher/mock_matcher.go | 56 + .../github.com/golang/mock/mockgen/mockgen.go | 442 + .../golang/mock/mockgen/model/model.go | 431 + .../github.com/golang/mock/mockgen/parse.go | 447 + .../github.com/golang/mock/mockgen/reflect.go | 163 + .../mockgen/tests/unexported_method/README.md | 1 + .../tests/unexported_method/bugreport.go | 15 + .../tests/unexported_method/bugreport_mock.go | 44 + .../tests/unexported_method/bugreport_test.go | 17 + .../github.com/golang/mock/sample/README.md | 16 + .../golang/mock/sample/imp1/imp1.go | 17 + .../golang/mock/sample/imp2/imp2.go | 3 + .../golang/mock/sample/imp3/imp3.go | 3 + .../golang/mock/sample/imp4/imp4.go | 3 + .../golang/mock/sample/mock_user/mock_user.go | 383 + vendor/github.com/golang/mock/sample/user.go | 114 + .../golang/mock/sample/user_test.go | 122 + vendor/github.com/mandykoh/keva/.gitignore | 1 + vendor/github.com/mandykoh/keva/.travis.yml | 9 + vendor/github.com/mandykoh/keva/LICENSE | 21 + vendor/github.com/mandykoh/keva/README.md | 7 + vendor/github.com/mandykoh/keva/bucket.go | 163 + .../github.com/mandykoh/keva/bucket_test.go | 232 + .../github.com/mandykoh/keva/bucketcache.go | 137 + .../mandykoh/keva/bucketcache_test.go | 253 + .../mandykoh/keva/bucketcacheentry.go | 23 + .../mandykoh/keva/bucketcachetrie.go | 71 + .../mandykoh/keva/bucketcachetrie_test.go | 139 + vendor/github.com/mandykoh/keva/bucketpath.go | 35 + .../mandykoh/keva/bucketpath_test.go | 63 + .../github.com/mandykoh/keva/common_test.go | 10 + vendor/github.com/mandykoh/keva/store.go | 170 + vendor/github.com/mandykoh/keva/store_test.go | 165 + vendor/github.com/mandykoh/keva/storeinfo.go | 6 + .../github.com/mandykoh/symlock/.travis.yml | 9 + vendor/github.com/mandykoh/symlock/LICENSE | 21 + vendor/github.com/mandykoh/symlock/README.md | 51 + .../github.com/mandykoh/symlock/partition.go | 8 + vendor/github.com/mandykoh/symlock/symlock.go | 67 + .../mandykoh/symlock/symlock_test.go | 113 + .../mandykoh/symlock/symlockentry.go | 8 + vendor/github.com/tslocum/markov/.gitignore | 1 + vendor/github.com/tslocum/markov/.travis.yml | 9 + vendor/github.com/tslocum/markov/LICENSE | 21 + vendor/github.com/tslocum/markov/README.md | 71 + .../github.com/tslocum/markov/accumulator.go | 21 + .../tslocum/markov/accumulator_test.go | 51 + .../tslocum/markov/bolttablestore.go | 70 + vendor/github.com/tslocum/markov/generator.go | 27 + .../tslocum/markov/generator_test.go | 55 + .../tslocum/markov/inmemorytablestore.go | 27 + .../tslocum/markov/kevatablestore.go | 36 + vendor/github.com/tslocum/markov/model.go | 46 + .../github.com/tslocum/markov/model_test.go | 80 + vendor/github.com/tslocum/markov/sequence.go | 51 + .../tslocum/markov/sequence_test.go | 93 + vendor/github.com/tslocum/markov/table.go | 63 + .../github.com/tslocum/markov/table_test.go | 136 + .../github.com/tslocum/markov/tableentry.go | 6 + .../github.com/tslocum/markov/tablestore.go | 10 + vendor/golang.org/x/net/.gitattributes | 10 + vendor/golang.org/x/net/.gitignore | 2 + vendor/golang.org/x/net/AUTHORS | 3 + vendor/golang.org/x/net/CONTRIBUTING.md | 31 + vendor/golang.org/x/net/CONTRIBUTORS | 3 + vendor/golang.org/x/net/LICENSE | 27 + vendor/golang.org/x/net/PATENTS | 22 + vendor/golang.org/x/net/README | 3 + vendor/golang.org/x/net/bpf/asm.go | 41 + vendor/golang.org/x/net/bpf/constants.go | 218 + vendor/golang.org/x/net/bpf/doc.go | 82 + vendor/golang.org/x/net/bpf/instructions.go | 704 + .../golang.org/x/net/bpf/instructions_test.go | 525 + vendor/golang.org/x/net/bpf/setter.go | 10 + .../x/net/bpf/testdata/all_instructions.bpf | 1 + .../x/net/bpf/testdata/all_instructions.txt | 79 + vendor/golang.org/x/net/bpf/vm.go | 140 + vendor/golang.org/x/net/bpf/vm_aluop_test.go | 512 + vendor/golang.org/x/net/bpf/vm_bpf_test.go | 192 + .../golang.org/x/net/bpf/vm_extension_test.go | 49 + .../golang.org/x/net/bpf/vm_instructions.go | 174 + vendor/golang.org/x/net/bpf/vm_jump_test.go | 380 + vendor/golang.org/x/net/bpf/vm_load_test.go | 246 + vendor/golang.org/x/net/bpf/vm_ret_test.go | 115 + .../golang.org/x/net/bpf/vm_scratch_test.go | 247 + vendor/golang.org/x/net/bpf/vm_test.go | 144 + vendor/golang.org/x/net/codereview.cfg | 1 + vendor/golang.org/x/net/context/context.go | 54 + .../golang.org/x/net/context/context_test.go | 583 + .../x/net/context/ctxhttp/ctxhttp.go | 74 + .../x/net/context/ctxhttp/ctxhttp_17_test.go | 29 + .../x/net/context/ctxhttp/ctxhttp_pre17.go | 147 + .../net/context/ctxhttp/ctxhttp_pre17_test.go | 79 + .../x/net/context/ctxhttp/ctxhttp_test.go | 105 + vendor/golang.org/x/net/context/go17.go | 72 + vendor/golang.org/x/net/context/go19.go | 20 + vendor/golang.org/x/net/context/pre_go17.go | 300 + vendor/golang.org/x/net/context/pre_go19.go | 109 + .../x/net/context/withtimeout_test.go | 31 + vendor/golang.org/x/net/dict/dict.go | 210 + .../x/net/dns/dnsmessage/example_test.go | 132 + .../x/net/dns/dnsmessage/message.go | 1997 ++ .../x/net/dns/dnsmessage/message_test.go | 1116 + vendor/golang.org/x/net/html/atom/atom.go | 78 + .../golang.org/x/net/html/atom/atom_test.go | 109 + vendor/golang.org/x/net/html/atom/gen.go | 648 + vendor/golang.org/x/net/html/atom/table.go | 713 + .../golang.org/x/net/html/atom/table_test.go | 351 + .../golang.org/x/net/html/charset/charset.go | 257 + .../x/net/html/charset/charset_test.go | 237 + .../html/charset/testdata/HTTP-charset.html | 48 + .../charset/testdata/HTTP-vs-UTF-8-BOM.html | 48 + .../testdata/HTTP-vs-meta-charset.html | 49 + .../testdata/HTTP-vs-meta-content.html | 49 + .../testdata/No-encoding-declaration.html | 47 + .../x/net/html/charset/testdata/README | 9 + .../html/charset/testdata/UTF-16BE-BOM.html | Bin 0 -> 2670 bytes .../html/charset/testdata/UTF-16LE-BOM.html | Bin 0 -> 2682 bytes .../testdata/UTF-8-BOM-vs-meta-charset.html | 49 + .../testdata/UTF-8-BOM-vs-meta-content.html | 48 + .../testdata/meta-charset-attribute.html | 48 + .../testdata/meta-content-attribute.html | 48 + vendor/golang.org/x/net/html/const.go | 102 + vendor/golang.org/x/net/html/doc.go | 106 + vendor/golang.org/x/net/html/doctype.go | 156 + vendor/golang.org/x/net/html/entity.go | 2253 +++ vendor/golang.org/x/net/html/entity_test.go | 29 + vendor/golang.org/x/net/html/escape.go | 258 + vendor/golang.org/x/net/html/escape_test.go | 97 + vendor/golang.org/x/net/html/example_test.go | 40 + vendor/golang.org/x/net/html/foreign.go | 226 + vendor/golang.org/x/net/html/node.go | 193 + vendor/golang.org/x/net/html/node_test.go | 146 + vendor/golang.org/x/net/html/parse.go | 2094 ++ vendor/golang.org/x/net/html/parse_test.go | 388 + vendor/golang.org/x/net/html/render.go | 271 + vendor/golang.org/x/net/html/render_test.go | 156 + .../golang.org/x/net/html/testdata/go1.html | 2237 +++ .../x/net/html/testdata/webkit/README | 28 + .../x/net/html/testdata/webkit/adoption01.dat | 194 + .../x/net/html/testdata/webkit/adoption02.dat | 31 + .../x/net/html/testdata/webkit/comments01.dat | 135 + .../x/net/html/testdata/webkit/doctype01.dat | 370 + .../x/net/html/testdata/webkit/entities01.dat | 603 + .../x/net/html/testdata/webkit/entities02.dat | 249 + .../html/testdata/webkit/html5test-com.dat | 246 + .../x/net/html/testdata/webkit/inbody01.dat | 43 + .../x/net/html/testdata/webkit/isindex.dat | 40 + ...pending-spec-changes-plain-text-unsafe.dat | Bin 0 -> 115 bytes .../testdata/webkit/pending-spec-changes.dat | 52 + .../testdata/webkit/plain-text-unsafe.dat | Bin 0 -> 4166 bytes .../net/html/testdata/webkit/scriptdata01.dat | 308 + .../testdata/webkit/scripted/adoption01.dat | 15 + .../testdata/webkit/scripted/webkit01.dat | 28 + .../x/net/html/testdata/webkit/tables01.dat | 212 + .../x/net/html/testdata/webkit/tests1.dat | 1952 ++ .../x/net/html/testdata/webkit/tests10.dat | 799 + .../x/net/html/testdata/webkit/tests11.dat | 482 + .../x/net/html/testdata/webkit/tests12.dat | 62 + .../x/net/html/testdata/webkit/tests14.dat | 74 + .../x/net/html/testdata/webkit/tests15.dat | 208 + .../x/net/html/testdata/webkit/tests16.dat | 2299 +++ .../x/net/html/testdata/webkit/tests17.dat | 153 + .../x/net/html/testdata/webkit/tests18.dat | 269 + .../x/net/html/testdata/webkit/tests19.dat | 1237 ++ .../x/net/html/testdata/webkit/tests2.dat | 763 + .../x/net/html/testdata/webkit/tests20.dat | 455 + .../x/net/html/testdata/webkit/tests21.dat | 221 + .../x/net/html/testdata/webkit/tests22.dat | 157 + .../x/net/html/testdata/webkit/tests23.dat | 155 + .../x/net/html/testdata/webkit/tests24.dat | 79 + .../x/net/html/testdata/webkit/tests25.dat | 219 + .../x/net/html/testdata/webkit/tests26.dat | 313 + .../x/net/html/testdata/webkit/tests3.dat | 305 + .../x/net/html/testdata/webkit/tests4.dat | 59 + .../x/net/html/testdata/webkit/tests5.dat | 191 + .../x/net/html/testdata/webkit/tests6.dat | 663 + .../x/net/html/testdata/webkit/tests7.dat | 390 + .../x/net/html/testdata/webkit/tests8.dat | 148 + .../x/net/html/testdata/webkit/tests9.dat | 457 + .../testdata/webkit/tests_innerHTML_1.dat | 741 + .../x/net/html/testdata/webkit/tricky01.dat | 261 + .../x/net/html/testdata/webkit/webkit01.dat | 610 + .../x/net/html/testdata/webkit/webkit02.dat | 159 + vendor/golang.org/x/net/html/token.go | 1219 ++ vendor/golang.org/x/net/html/token_test.go | 748 + vendor/golang.org/x/net/http2/.gitignore | 2 + vendor/golang.org/x/net/http2/Dockerfile | 51 + vendor/golang.org/x/net/http2/Makefile | 3 + vendor/golang.org/x/net/http2/README | 20 + vendor/golang.org/x/net/http2/ciphers.go | 641 + vendor/golang.org/x/net/http2/ciphers_test.go | 309 + .../x/net/http2/client_conn_pool.go | 256 + .../x/net/http2/configure_transport.go | 80 + vendor/golang.org/x/net/http2/databuffer.go | 146 + .../golang.org/x/net/http2/databuffer_test.go | 157 + vendor/golang.org/x/net/http2/errors.go | 133 + vendor/golang.org/x/net/http2/errors_test.go | 24 + vendor/golang.org/x/net/http2/flow.go | 50 + vendor/golang.org/x/net/http2/flow_test.go | 53 + vendor/golang.org/x/net/http2/frame.go | 1579 ++ vendor/golang.org/x/net/http2/frame_test.go | 1191 ++ vendor/golang.org/x/net/http2/go16.go | 16 + vendor/golang.org/x/net/http2/go17.go | 106 + vendor/golang.org/x/net/http2/go17_not18.go | 36 + vendor/golang.org/x/net/http2/go18.go | 56 + vendor/golang.org/x/net/http2/go18_test.go | 79 + vendor/golang.org/x/net/http2/go19.go | 16 + vendor/golang.org/x/net/http2/go19_test.go | 60 + vendor/golang.org/x/net/http2/gotrack.go | 170 + vendor/golang.org/x/net/http2/gotrack_test.go | 33 + .../golang.org/x/net/http2/h2demo/.gitignore | 5 + vendor/golang.org/x/net/http2/h2demo/Makefile | 8 + vendor/golang.org/x/net/http2/h2demo/README | 16 + .../golang.org/x/net/http2/h2demo/h2demo.go | 538 + .../golang.org/x/net/http2/h2demo/launch.go | 302 + .../golang.org/x/net/http2/h2demo/rootCA.key | 27 + .../golang.org/x/net/http2/h2demo/rootCA.pem | 26 + .../golang.org/x/net/http2/h2demo/rootCA.srl | 1 + .../golang.org/x/net/http2/h2demo/server.crt | 20 + .../golang.org/x/net/http2/h2demo/server.key | 27 + vendor/golang.org/x/net/http2/h2demo/tmpl.go | 1991 ++ vendor/golang.org/x/net/http2/h2i/README.md | 97 + vendor/golang.org/x/net/http2/h2i/h2i.go | 522 + vendor/golang.org/x/net/http2/headermap.go | 78 + vendor/golang.org/x/net/http2/hpack/encode.go | 240 + .../x/net/http2/hpack/encode_test.go | 386 + vendor/golang.org/x/net/http2/hpack/hpack.go | 490 + .../x/net/http2/hpack/hpack_test.go | 722 + .../golang.org/x/net/http2/hpack/huffman.go | 212 + vendor/golang.org/x/net/http2/hpack/tables.go | 479 + .../x/net/http2/hpack/tables_test.go | 214 + vendor/golang.org/x/net/http2/http2.go | 391 + vendor/golang.org/x/net/http2/http2_test.go | 199 + vendor/golang.org/x/net/http2/not_go16.go | 21 + vendor/golang.org/x/net/http2/not_go17.go | 87 + vendor/golang.org/x/net/http2/not_go18.go | 29 + vendor/golang.org/x/net/http2/not_go19.go | 16 + vendor/golang.org/x/net/http2/pipe.go | 163 + vendor/golang.org/x/net/http2/pipe_test.go | 130 + vendor/golang.org/x/net/http2/server.go | 2857 +++ .../x/net/http2/server_push_test.go | 521 + vendor/golang.org/x/net/http2/server_test.go | 3721 ++++ .../testdata/draft-ietf-httpbis-http2.xml | 5021 +++++ vendor/golang.org/x/net/http2/transport.go | 2275 +++ .../golang.org/x/net/http2/transport_test.go | 3686 ++++ vendor/golang.org/x/net/http2/write.go | 370 + vendor/golang.org/x/net/http2/writesched.go | 242 + .../x/net/http2/writesched_priority.go | 452 + .../x/net/http2/writesched_priority_test.go | 541 + .../x/net/http2/writesched_random.go | 72 + .../x/net/http2/writesched_random_test.go | 44 + .../golang.org/x/net/http2/writesched_test.go | 125 + vendor/golang.org/x/net/http2/z_spec_test.go | 356 + vendor/golang.org/x/net/icmp/dstunreach.go | 41 + vendor/golang.org/x/net/icmp/echo.go | 45 + vendor/golang.org/x/net/icmp/endpoint.go | 113 + vendor/golang.org/x/net/icmp/example_test.go | 63 + vendor/golang.org/x/net/icmp/extension.go | 89 + .../golang.org/x/net/icmp/extension_test.go | 259 + vendor/golang.org/x/net/icmp/helper_posix.go | 75 + vendor/golang.org/x/net/icmp/interface.go | 236 + vendor/golang.org/x/net/icmp/ipv4.go | 61 + vendor/golang.org/x/net/icmp/ipv4_test.go | 83 + vendor/golang.org/x/net/icmp/ipv6.go | 23 + vendor/golang.org/x/net/icmp/listen_posix.go | 100 + vendor/golang.org/x/net/icmp/listen_stub.go | 33 + vendor/golang.org/x/net/icmp/message.go | 152 + vendor/golang.org/x/net/icmp/message_test.go | 134 + vendor/golang.org/x/net/icmp/messagebody.go | 41 + vendor/golang.org/x/net/icmp/mpls.go | 77 + vendor/golang.org/x/net/icmp/multipart.go | 109 + .../golang.org/x/net/icmp/multipart_test.go | 442 + vendor/golang.org/x/net/icmp/packettoobig.go | 43 + vendor/golang.org/x/net/icmp/paramprob.go | 63 + vendor/golang.org/x/net/icmp/ping_test.go | 200 + vendor/golang.org/x/net/icmp/sys_freebsd.go | 11 + vendor/golang.org/x/net/icmp/timeexceeded.go | 39 + vendor/golang.org/x/net/idna/example_test.go | 70 + vendor/golang.org/x/net/idna/idna.go | 680 + vendor/golang.org/x/net/idna/idna_test.go | 108 + vendor/golang.org/x/net/idna/punycode.go | 203 + vendor/golang.org/x/net/idna/punycode_test.go | 198 + vendor/golang.org/x/net/idna/tables.go | 4477 +++++ vendor/golang.org/x/net/idna/trie.go | 72 + vendor/golang.org/x/net/idna/trieval.go | 114 + .../golang.org/x/net/internal/iana/const.go | 180 + vendor/golang.org/x/net/internal/iana/gen.go | 293 + .../x/net/internal/nettest/helper_bsd.go | 53 + .../x/net/internal/nettest/helper_nobsd.go | 15 + .../x/net/internal/nettest/helper_posix.go | 31 + .../x/net/internal/nettest/helper_stub.go | 32 + .../x/net/internal/nettest/helper_unix.go | 29 + .../x/net/internal/nettest/helper_windows.go | 42 + .../x/net/internal/nettest/interface.go | 94 + .../x/net/internal/nettest/rlimit.go | 11 + .../x/net/internal/nettest/stack.go | 147 + .../x/net/internal/socket/cmsghdr.go | 11 + .../x/net/internal/socket/cmsghdr_bsd.go | 13 + .../internal/socket/cmsghdr_linux_32bit.go | 14 + .../internal/socket/cmsghdr_linux_64bit.go | 14 + .../internal/socket/cmsghdr_solaris_64bit.go | 14 + .../x/net/internal/socket/cmsghdr_stub.go | 17 + .../x/net/internal/socket/defs_darwin.go | 44 + .../x/net/internal/socket/defs_dragonfly.go | 44 + .../x/net/internal/socket/defs_freebsd.go | 44 + .../x/net/internal/socket/defs_linux.go | 49 + .../x/net/internal/socket/defs_netbsd.go | 47 + .../x/net/internal/socket/defs_openbsd.go | 44 + .../x/net/internal/socket/defs_solaris.go | 44 + .../x/net/internal/socket/error_unix.go | 31 + .../x/net/internal/socket/error_windows.go | 26 + .../x/net/internal/socket/iovec_32bit.go | 15 + .../x/net/internal/socket/iovec_64bit.go | 15 + .../internal/socket/iovec_solaris_64bit.go | 15 + .../x/net/internal/socket/iovec_stub.go | 11 + .../x/net/internal/socket/mmsghdr_stub.go | 21 + .../x/net/internal/socket/mmsghdr_unix.go | 42 + .../x/net/internal/socket/msghdr_bsd.go | 39 + .../x/net/internal/socket/msghdr_bsdvar.go | 12 + .../x/net/internal/socket/msghdr_linux.go | 36 + .../net/internal/socket/msghdr_linux_32bit.go | 20 + .../net/internal/socket/msghdr_linux_64bit.go | 20 + .../x/net/internal/socket/msghdr_openbsd.go | 10 + .../internal/socket/msghdr_solaris_64bit.go | 34 + .../x/net/internal/socket/msghdr_stub.go | 14 + .../x/net/internal/socket/rawconn.go | 66 + .../x/net/internal/socket/rawconn_mmsg.go | 74 + .../x/net/internal/socket/rawconn_msg.go | 77 + .../x/net/internal/socket/rawconn_nommsg.go | 18 + .../x/net/internal/socket/rawconn_nomsg.go | 18 + .../x/net/internal/socket/rawconn_stub.go | 25 + .../x/net/internal/socket/reflect.go | 62 + .../x/net/internal/socket/socket.go | 285 + .../net/internal/socket/socket_go1_9_test.go | 256 + .../x/net/internal/socket/socket_test.go | 46 + .../golang.org/x/net/internal/socket/sys.go | 33 + .../x/net/internal/socket/sys_bsd.go | 17 + .../x/net/internal/socket/sys_bsdvar.go | 14 + .../x/net/internal/socket/sys_darwin.go | 7 + .../x/net/internal/socket/sys_dragonfly.go | 7 + .../x/net/internal/socket/sys_linux.go | 27 + .../x/net/internal/socket/sys_linux_386.go | 55 + .../x/net/internal/socket/sys_linux_386.s | 11 + .../x/net/internal/socket/sys_linux_amd64.go | 10 + .../x/net/internal/socket/sys_linux_arm.go | 10 + .../x/net/internal/socket/sys_linux_arm64.go | 10 + .../x/net/internal/socket/sys_linux_mips.go | 10 + .../x/net/internal/socket/sys_linux_mips64.go | 10 + .../net/internal/socket/sys_linux_mips64le.go | 10 + .../x/net/internal/socket/sys_linux_mipsle.go | 10 + .../x/net/internal/socket/sys_linux_ppc64.go | 10 + .../net/internal/socket/sys_linux_ppc64le.go | 10 + .../x/net/internal/socket/sys_linux_s390x.go | 55 + .../x/net/internal/socket/sys_linux_s390x.s | 11 + .../x/net/internal/socket/sys_netbsd.go | 25 + .../x/net/internal/socket/sys_posix.go | 168 + .../x/net/internal/socket/sys_solaris.go | 71 + .../x/net/internal/socket/sys_solaris_amd64.s | 11 + .../x/net/internal/socket/sys_stub.go | 64 + .../x/net/internal/socket/sys_unix.go | 33 + .../x/net/internal/socket/sys_windows.go | 70 + .../x/net/internal/socket/zsys_darwin_386.go | 59 + .../net/internal/socket/zsys_darwin_amd64.go | 61 + .../x/net/internal/socket/zsys_darwin_arm.go | 59 + .../internal/socket/zsys_dragonfly_amd64.go | 61 + .../x/net/internal/socket/zsys_freebsd_386.go | 59 + .../net/internal/socket/zsys_freebsd_amd64.go | 61 + .../x/net/internal/socket/zsys_freebsd_arm.go | 59 + .../x/net/internal/socket/zsys_linux_386.go | 63 + .../x/net/internal/socket/zsys_linux_amd64.go | 66 + .../x/net/internal/socket/zsys_linux_arm.go | 63 + .../x/net/internal/socket/zsys_linux_arm64.go | 66 + .../x/net/internal/socket/zsys_linux_mips.go | 63 + .../net/internal/socket/zsys_linux_mips64.go | 66 + .../internal/socket/zsys_linux_mips64le.go | 66 + .../net/internal/socket/zsys_linux_mipsle.go | 63 + .../x/net/internal/socket/zsys_linux_ppc64.go | 66 + .../net/internal/socket/zsys_linux_ppc64le.go | 66 + .../x/net/internal/socket/zsys_linux_s390x.go | 66 + .../x/net/internal/socket/zsys_netbsd_386.go | 65 + .../net/internal/socket/zsys_netbsd_amd64.go | 68 + .../x/net/internal/socket/zsys_netbsd_arm.go | 59 + .../x/net/internal/socket/zsys_openbsd_386.go | 59 + .../net/internal/socket/zsys_openbsd_amd64.go | 61 + .../x/net/internal/socket/zsys_openbsd_arm.go | 59 + .../net/internal/socket/zsys_solaris_amd64.go | 60 + .../x/net/internal/timeseries/timeseries.go | 525 + .../internal/timeseries/timeseries_test.go | 170 + vendor/golang.org/x/net/ipv4/batch.go | 191 + vendor/golang.org/x/net/ipv4/bpf_test.go | 93 + vendor/golang.org/x/net/ipv4/control.go | 144 + vendor/golang.org/x/net/ipv4/control_bsd.go | 40 + .../golang.org/x/net/ipv4/control_pktinfo.go | 39 + vendor/golang.org/x/net/ipv4/control_stub.go | 13 + vendor/golang.org/x/net/ipv4/control_test.go | 21 + vendor/golang.org/x/net/ipv4/control_unix.go | 73 + .../golang.org/x/net/ipv4/control_windows.go | 16 + vendor/golang.org/x/net/ipv4/defs_darwin.go | 77 + .../golang.org/x/net/ipv4/defs_dragonfly.go | 38 + vendor/golang.org/x/net/ipv4/defs_freebsd.go | 75 + vendor/golang.org/x/net/ipv4/defs_linux.go | 122 + vendor/golang.org/x/net/ipv4/defs_netbsd.go | 37 + vendor/golang.org/x/net/ipv4/defs_openbsd.go | 37 + vendor/golang.org/x/net/ipv4/defs_solaris.go | 84 + vendor/golang.org/x/net/ipv4/dgramopt.go | 265 + vendor/golang.org/x/net/ipv4/doc.go | 244 + vendor/golang.org/x/net/ipv4/endpoint.go | 187 + vendor/golang.org/x/net/ipv4/example_test.go | 224 + vendor/golang.org/x/net/ipv4/gen.go | 199 + vendor/golang.org/x/net/ipv4/genericopt.go | 57 + vendor/golang.org/x/net/ipv4/header.go | 159 + vendor/golang.org/x/net/ipv4/header_test.go | 228 + vendor/golang.org/x/net/ipv4/helper.go | 63 + vendor/golang.org/x/net/ipv4/iana.go | 34 + vendor/golang.org/x/net/ipv4/icmp.go | 57 + vendor/golang.org/x/net/ipv4/icmp_linux.go | 25 + vendor/golang.org/x/net/ipv4/icmp_stub.go | 25 + vendor/golang.org/x/net/ipv4/icmp_test.go | 95 + .../golang.org/x/net/ipv4/multicast_test.go | 334 + .../x/net/ipv4/multicastlistener_test.go | 265 + .../x/net/ipv4/multicastsockopt_test.go | 195 + vendor/golang.org/x/net/ipv4/packet.go | 69 + vendor/golang.org/x/net/ipv4/packet_go1_8.go | 56 + vendor/golang.org/x/net/ipv4/packet_go1_9.go | 67 + vendor/golang.org/x/net/ipv4/payload.go | 23 + vendor/golang.org/x/net/ipv4/payload_cmsg.go | 36 + .../x/net/ipv4/payload_cmsg_go1_8.go | 59 + .../x/net/ipv4/payload_cmsg_go1_9.go | 67 + .../golang.org/x/net/ipv4/payload_nocmsg.go | 42 + .../x/net/ipv4/readwrite_go1_8_test.go | 248 + .../x/net/ipv4/readwrite_go1_9_test.go | 388 + .../golang.org/x/net/ipv4/readwrite_test.go | 140 + vendor/golang.org/x/net/ipv4/sockopt.go | 44 + vendor/golang.org/x/net/ipv4/sockopt_posix.go | 71 + vendor/golang.org/x/net/ipv4/sockopt_stub.go | 42 + vendor/golang.org/x/net/ipv4/sys_asmreq.go | 119 + .../golang.org/x/net/ipv4/sys_asmreq_stub.go | 25 + vendor/golang.org/x/net/ipv4/sys_asmreqn.go | 42 + .../golang.org/x/net/ipv4/sys_asmreqn_stub.go | 21 + vendor/golang.org/x/net/ipv4/sys_bpf.go | 23 + vendor/golang.org/x/net/ipv4/sys_bpf_stub.go | 16 + vendor/golang.org/x/net/ipv4/sys_bsd.go | 37 + vendor/golang.org/x/net/ipv4/sys_darwin.go | 93 + vendor/golang.org/x/net/ipv4/sys_dragonfly.go | 35 + vendor/golang.org/x/net/ipv4/sys_freebsd.go | 76 + vendor/golang.org/x/net/ipv4/sys_linux.go | 59 + vendor/golang.org/x/net/ipv4/sys_solaris.go | 57 + vendor/golang.org/x/net/ipv4/sys_ssmreq.go | 54 + .../golang.org/x/net/ipv4/sys_ssmreq_stub.go | 21 + vendor/golang.org/x/net/ipv4/sys_stub.go | 13 + vendor/golang.org/x/net/ipv4/sys_windows.go | 67 + vendor/golang.org/x/net/ipv4/unicast_test.go | 247 + .../x/net/ipv4/unicastsockopt_test.go | 148 + vendor/golang.org/x/net/ipv4/zsys_darwin.go | 99 + .../golang.org/x/net/ipv4/zsys_dragonfly.go | 31 + .../golang.org/x/net/ipv4/zsys_freebsd_386.go | 93 + .../x/net/ipv4/zsys_freebsd_amd64.go | 95 + .../golang.org/x/net/ipv4/zsys_freebsd_arm.go | 95 + .../golang.org/x/net/ipv4/zsys_linux_386.go | 148 + .../golang.org/x/net/ipv4/zsys_linux_amd64.go | 150 + .../golang.org/x/net/ipv4/zsys_linux_arm.go | 148 + .../golang.org/x/net/ipv4/zsys_linux_arm64.go | 150 + .../golang.org/x/net/ipv4/zsys_linux_mips.go | 148 + .../x/net/ipv4/zsys_linux_mips64.go | 150 + .../x/net/ipv4/zsys_linux_mips64le.go | 150 + .../x/net/ipv4/zsys_linux_mipsle.go | 148 + .../golang.org/x/net/ipv4/zsys_linux_ppc.go | 148 + .../golang.org/x/net/ipv4/zsys_linux_ppc64.go | 150 + .../x/net/ipv4/zsys_linux_ppc64le.go | 150 + .../golang.org/x/net/ipv4/zsys_linux_s390x.go | 150 + vendor/golang.org/x/net/ipv4/zsys_netbsd.go | 30 + vendor/golang.org/x/net/ipv4/zsys_openbsd.go | 30 + vendor/golang.org/x/net/ipv4/zsys_solaris.go | 100 + vendor/golang.org/x/net/ipv6/batch.go | 119 + vendor/golang.org/x/net/ipv6/bpf_test.go | 96 + vendor/golang.org/x/net/ipv6/control.go | 187 + .../x/net/ipv6/control_rfc2292_unix.go | 48 + .../x/net/ipv6/control_rfc3542_unix.go | 94 + vendor/golang.org/x/net/ipv6/control_stub.go | 13 + vendor/golang.org/x/net/ipv6/control_test.go | 21 + vendor/golang.org/x/net/ipv6/control_unix.go | 55 + .../golang.org/x/net/ipv6/control_windows.go | 16 + vendor/golang.org/x/net/ipv6/defs_darwin.go | 112 + .../golang.org/x/net/ipv6/defs_dragonfly.go | 84 + vendor/golang.org/x/net/ipv6/defs_freebsd.go | 105 + vendor/golang.org/x/net/ipv6/defs_linux.go | 147 + vendor/golang.org/x/net/ipv6/defs_netbsd.go | 80 + vendor/golang.org/x/net/ipv6/defs_openbsd.go | 89 + vendor/golang.org/x/net/ipv6/defs_solaris.go | 114 + vendor/golang.org/x/net/ipv6/dgramopt.go | 302 + vendor/golang.org/x/net/ipv6/doc.go | 243 + vendor/golang.org/x/net/ipv6/endpoint.go | 128 + vendor/golang.org/x/net/ipv6/example_test.go | 216 + vendor/golang.org/x/net/ipv6/gen.go | 199 + vendor/golang.org/x/net/ipv6/genericopt.go | 58 + vendor/golang.org/x/net/ipv6/header.go | 55 + vendor/golang.org/x/net/ipv6/header_test.go | 55 + vendor/golang.org/x/net/ipv6/helper.go | 57 + vendor/golang.org/x/net/ipv6/iana.go | 82 + vendor/golang.org/x/net/ipv6/icmp.go | 60 + vendor/golang.org/x/net/ipv6/icmp_bsd.go | 29 + vendor/golang.org/x/net/ipv6/icmp_linux.go | 27 + vendor/golang.org/x/net/ipv6/icmp_solaris.go | 27 + vendor/golang.org/x/net/ipv6/icmp_stub.go | 23 + vendor/golang.org/x/net/ipv6/icmp_test.go | 96 + vendor/golang.org/x/net/ipv6/icmp_windows.go | 22 + .../x/net/ipv6/mocktransponder_test.go | 32 + .../golang.org/x/net/ipv6/multicast_test.go | 264 + .../x/net/ipv6/multicastlistener_test.go | 261 + .../x/net/ipv6/multicastsockopt_test.go | 157 + vendor/golang.org/x/net/ipv6/payload.go | 23 + vendor/golang.org/x/net/ipv6/payload_cmsg.go | 35 + .../x/net/ipv6/payload_cmsg_go1_8.go | 55 + .../x/net/ipv6/payload_cmsg_go1_9.go | 57 + .../golang.org/x/net/ipv6/payload_nocmsg.go | 41 + .../x/net/ipv6/readwrite_go1_8_test.go | 242 + .../x/net/ipv6/readwrite_go1_9_test.go | 373 + .../golang.org/x/net/ipv6/readwrite_test.go | 148 + vendor/golang.org/x/net/ipv6/sockopt.go | 43 + vendor/golang.org/x/net/ipv6/sockopt_posix.go | 87 + vendor/golang.org/x/net/ipv6/sockopt_stub.go | 46 + vendor/golang.org/x/net/ipv6/sockopt_test.go | 133 + vendor/golang.org/x/net/ipv6/sys_asmreq.go | 24 + .../golang.org/x/net/ipv6/sys_asmreq_stub.go | 17 + vendor/golang.org/x/net/ipv6/sys_bpf.go | 23 + vendor/golang.org/x/net/ipv6/sys_bpf_stub.go | 16 + vendor/golang.org/x/net/ipv6/sys_bsd.go | 57 + vendor/golang.org/x/net/ipv6/sys_darwin.go | 106 + vendor/golang.org/x/net/ipv6/sys_freebsd.go | 92 + vendor/golang.org/x/net/ipv6/sys_linux.go | 74 + vendor/golang.org/x/net/ipv6/sys_solaris.go | 74 + vendor/golang.org/x/net/ipv6/sys_ssmreq.go | 54 + .../golang.org/x/net/ipv6/sys_ssmreq_stub.go | 21 + vendor/golang.org/x/net/ipv6/sys_stub.go | 13 + vendor/golang.org/x/net/ipv6/sys_windows.go | 75 + vendor/golang.org/x/net/ipv6/unicast_test.go | 184 + .../x/net/ipv6/unicastsockopt_test.go | 120 + vendor/golang.org/x/net/ipv6/zsys_darwin.go | 131 + .../golang.org/x/net/ipv6/zsys_dragonfly.go | 88 + .../golang.org/x/net/ipv6/zsys_freebsd_386.go | 122 + .../x/net/ipv6/zsys_freebsd_amd64.go | 124 + .../golang.org/x/net/ipv6/zsys_freebsd_arm.go | 124 + .../golang.org/x/net/ipv6/zsys_linux_386.go | 170 + .../golang.org/x/net/ipv6/zsys_linux_amd64.go | 172 + .../golang.org/x/net/ipv6/zsys_linux_arm.go | 170 + .../golang.org/x/net/ipv6/zsys_linux_arm64.go | 172 + .../golang.org/x/net/ipv6/zsys_linux_mips.go | 170 + .../x/net/ipv6/zsys_linux_mips64.go | 172 + .../x/net/ipv6/zsys_linux_mips64le.go | 172 + .../x/net/ipv6/zsys_linux_mipsle.go | 170 + .../golang.org/x/net/ipv6/zsys_linux_ppc.go | 170 + .../golang.org/x/net/ipv6/zsys_linux_ppc64.go | 172 + .../x/net/ipv6/zsys_linux_ppc64le.go | 172 + .../golang.org/x/net/ipv6/zsys_linux_s390x.go | 172 + vendor/golang.org/x/net/ipv6/zsys_netbsd.go | 84 + vendor/golang.org/x/net/ipv6/zsys_openbsd.go | 93 + vendor/golang.org/x/net/ipv6/zsys_solaris.go | 131 + .../golang.org/x/net/lex/httplex/httplex.go | 351 + .../x/net/lex/httplex/httplex_test.go | 119 + vendor/golang.org/x/net/lif/address.go | 105 + vendor/golang.org/x/net/lif/address_test.go | 123 + vendor/golang.org/x/net/lif/binary.go | 115 + vendor/golang.org/x/net/lif/defs_solaris.go | 90 + vendor/golang.org/x/net/lif/lif.go | 43 + vendor/golang.org/x/net/lif/link.go | 126 + vendor/golang.org/x/net/lif/link_test.go | 63 + vendor/golang.org/x/net/lif/sys.go | 21 + .../golang.org/x/net/lif/sys_solaris_amd64.s | 8 + vendor/golang.org/x/net/lif/syscall.go | 28 + .../x/net/lif/zsys_solaris_amd64.go | 103 + vendor/golang.org/x/net/nettest/conntest.go | 456 + .../golang.org/x/net/nettest/conntest_go16.go | 24 + .../golang.org/x/net/nettest/conntest_go17.go | 24 + .../golang.org/x/net/nettest/conntest_test.go | 76 + vendor/golang.org/x/net/netutil/listen.go | 48 + .../golang.org/x/net/netutil/listen_test.go | 101 + vendor/golang.org/x/net/proxy/direct.go | 18 + vendor/golang.org/x/net/proxy/per_host.go | 140 + .../golang.org/x/net/proxy/per_host_test.go | 55 + vendor/golang.org/x/net/proxy/proxy.go | 134 + vendor/golang.org/x/net/proxy/proxy_test.go | 215 + vendor/golang.org/x/net/proxy/socks5.go | 213 + vendor/golang.org/x/net/publicsuffix/gen.go | 713 + vendor/golang.org/x/net/publicsuffix/list.go | 135 + .../x/net/publicsuffix/list_test.go | 416 + vendor/golang.org/x/net/publicsuffix/table.go | 9419 +++++++++ .../x/net/publicsuffix/table_test.go | 16756 ++++++++++++++++ vendor/golang.org/x/net/route/address.go | 425 + .../x/net/route/address_darwin_test.go | 63 + vendor/golang.org/x/net/route/address_test.go | 103 + vendor/golang.org/x/net/route/binary.go | 90 + vendor/golang.org/x/net/route/defs_darwin.go | 114 + .../golang.org/x/net/route/defs_dragonfly.go | 113 + vendor/golang.org/x/net/route/defs_freebsd.go | 337 + vendor/golang.org/x/net/route/defs_netbsd.go | 112 + vendor/golang.org/x/net/route/defs_openbsd.go | 116 + vendor/golang.org/x/net/route/interface.go | 64 + .../x/net/route/interface_announce.go | 32 + .../x/net/route/interface_classic.go | 66 + .../x/net/route/interface_freebsd.go | 78 + .../x/net/route/interface_multicast.go | 30 + .../x/net/route/interface_openbsd.go | 90 + vendor/golang.org/x/net/route/message.go | 72 + .../x/net/route/message_darwin_test.go | 34 + .../x/net/route/message_freebsd_test.go | 92 + vendor/golang.org/x/net/route/message_test.go | 239 + vendor/golang.org/x/net/route/route.go | 123 + .../golang.org/x/net/route/route_classic.go | 67 + .../golang.org/x/net/route/route_openbsd.go | 65 + vendor/golang.org/x/net/route/route_test.go | 390 + vendor/golang.org/x/net/route/sys.go | 39 + vendor/golang.org/x/net/route/sys_darwin.go | 87 + .../golang.org/x/net/route/sys_dragonfly.go | 76 + vendor/golang.org/x/net/route/sys_freebsd.go | 155 + vendor/golang.org/x/net/route/sys_netbsd.go | 71 + vendor/golang.org/x/net/route/sys_openbsd.go | 80 + vendor/golang.org/x/net/route/syscall.go | 28 + vendor/golang.org/x/net/route/zsys_darwin.go | 99 + .../golang.org/x/net/route/zsys_dragonfly.go | 98 + .../x/net/route/zsys_freebsd_386.go | 126 + .../x/net/route/zsys_freebsd_amd64.go | 123 + .../x/net/route/zsys_freebsd_arm.go | 123 + vendor/golang.org/x/net/route/zsys_netbsd.go | 97 + vendor/golang.org/x/net/route/zsys_openbsd.go | 101 + vendor/golang.org/x/net/trace/events.go | 532 + vendor/golang.org/x/net/trace/histogram.go | 365 + .../golang.org/x/net/trace/histogram_test.go | 325 + vendor/golang.org/x/net/trace/trace.go | 1082 + vendor/golang.org/x/net/trace/trace_go16.go | 21 + vendor/golang.org/x/net/trace/trace_go17.go | 21 + vendor/golang.org/x/net/trace/trace_test.go | 178 + vendor/golang.org/x/net/webdav/file.go | 796 + vendor/golang.org/x/net/webdav/file_go1.6.go | 17 + vendor/golang.org/x/net/webdav/file_go1.7.go | 16 + vendor/golang.org/x/net/webdav/file_test.go | 1184 ++ vendor/golang.org/x/net/webdav/if.go | 173 + vendor/golang.org/x/net/webdav/if_test.go | 322 + .../x/net/webdav/internal/xml/README | 11 + .../x/net/webdav/internal/xml/atom_test.go | 56 + .../x/net/webdav/internal/xml/example_test.go | 151 + .../x/net/webdav/internal/xml/marshal.go | 1223 ++ .../x/net/webdav/internal/xml/marshal_test.go | 1939 ++ .../x/net/webdav/internal/xml/read.go | 692 + .../x/net/webdav/internal/xml/read_test.go | 744 + .../x/net/webdav/internal/xml/typeinfo.go | 371 + .../x/net/webdav/internal/xml/xml.go | 1998 ++ .../x/net/webdav/internal/xml/xml_test.go | 752 + .../x/net/webdav/litmus_test_server.go | 94 + vendor/golang.org/x/net/webdav/lock.go | 445 + vendor/golang.org/x/net/webdav/lock_test.go | 731 + vendor/golang.org/x/net/webdav/prop.go | 418 + vendor/golang.org/x/net/webdav/prop_test.go | 613 + vendor/golang.org/x/net/webdav/webdav.go | 702 + vendor/golang.org/x/net/webdav/webdav_test.go | 344 + vendor/golang.org/x/net/webdav/xml.go | 519 + vendor/golang.org/x/net/webdav/xml_test.go | 906 + vendor/golang.org/x/net/websocket/client.go | 106 + vendor/golang.org/x/net/websocket/dial.go | 24 + .../golang.org/x/net/websocket/dial_test.go | 43 + .../x/net/websocket/exampledial_test.go | 31 + .../x/net/websocket/examplehandler_test.go | 26 + vendor/golang.org/x/net/websocket/hybi.go | 583 + .../golang.org/x/net/websocket/hybi_test.go | 608 + vendor/golang.org/x/net/websocket/server.go | 113 + .../golang.org/x/net/websocket/websocket.go | 448 + .../x/net/websocket/websocket_test.go | 665 + vendor/golang.org/x/net/xsrftoken/xsrf.go | 94 + .../golang.org/x/net/xsrftoken/xsrf_test.go | 83 + vendor/golang.org/x/sys/.gitattributes | 10 + vendor/golang.org/x/sys/.gitignore | 2 + vendor/golang.org/x/sys/AUTHORS | 3 + vendor/golang.org/x/sys/CONTRIBUTING.md | 31 + vendor/golang.org/x/sys/CONTRIBUTORS | 3 + vendor/golang.org/x/sys/LICENSE | 27 + vendor/golang.org/x/sys/PATENTS | 22 + vendor/golang.org/x/sys/README | 3 + vendor/golang.org/x/sys/codereview.cfg | 1 + vendor/golang.org/x/sys/plan9/asm.s | 8 + vendor/golang.org/x/sys/plan9/asm_plan9_386.s | 30 + .../golang.org/x/sys/plan9/asm_plan9_amd64.s | 30 + vendor/golang.org/x/sys/plan9/const_plan9.go | 70 + vendor/golang.org/x/sys/plan9/dir_plan9.go | 212 + vendor/golang.org/x/sys/plan9/env_plan9.go | 27 + vendor/golang.org/x/sys/plan9/env_unset.go | 14 + vendor/golang.org/x/sys/plan9/errors_plan9.go | 50 + vendor/golang.org/x/sys/plan9/mksyscall.pl | 319 + .../golang.org/x/sys/plan9/pwd_go15_plan9.go | 21 + vendor/golang.org/x/sys/plan9/pwd_plan9.go | 23 + vendor/golang.org/x/sys/plan9/race.go | 30 + vendor/golang.org/x/sys/plan9/race0.go | 25 + vendor/golang.org/x/sys/plan9/str.go | 22 + vendor/golang.org/x/sys/plan9/syscall.go | 74 + .../golang.org/x/sys/plan9/syscall_plan9.go | 349 + vendor/golang.org/x/sys/plan9/syscall_test.go | 33 + .../x/sys/plan9/zsyscall_plan9_386.go | 292 + .../x/sys/plan9/zsyscall_plan9_amd64.go | 292 + .../golang.org/x/sys/plan9/zsysnum_plan9.go | 49 + vendor/golang.org/x/sys/unix/.gitignore | 1 + vendor/golang.org/x/sys/unix/README.md | 173 + vendor/golang.org/x/sys/unix/asm_darwin_386.s | 29 + .../golang.org/x/sys/unix/asm_darwin_amd64.s | 29 + vendor/golang.org/x/sys/unix/asm_darwin_arm.s | 30 + .../golang.org/x/sys/unix/asm_darwin_arm64.s | 30 + .../x/sys/unix/asm_dragonfly_amd64.s | 29 + .../golang.org/x/sys/unix/asm_freebsd_386.s | 29 + .../golang.org/x/sys/unix/asm_freebsd_amd64.s | 29 + .../golang.org/x/sys/unix/asm_freebsd_arm.s | 29 + vendor/golang.org/x/sys/unix/asm_linux_386.s | 35 + .../golang.org/x/sys/unix/asm_linux_amd64.s | 29 + vendor/golang.org/x/sys/unix/asm_linux_arm.s | 29 + .../golang.org/x/sys/unix/asm_linux_arm64.s | 24 + .../golang.org/x/sys/unix/asm_linux_mips64x.s | 28 + .../golang.org/x/sys/unix/asm_linux_mipsx.s | 31 + .../golang.org/x/sys/unix/asm_linux_ppc64x.s | 28 + .../golang.org/x/sys/unix/asm_linux_s390x.s | 28 + vendor/golang.org/x/sys/unix/asm_netbsd_386.s | 29 + .../golang.org/x/sys/unix/asm_netbsd_amd64.s | 29 + vendor/golang.org/x/sys/unix/asm_netbsd_arm.s | 29 + .../golang.org/x/sys/unix/asm_openbsd_386.s | 29 + .../golang.org/x/sys/unix/asm_openbsd_amd64.s | 29 + .../golang.org/x/sys/unix/asm_openbsd_arm.s | 29 + .../golang.org/x/sys/unix/asm_solaris_amd64.s | 17 + .../golang.org/x/sys/unix/bluetooth_linux.go | 35 + vendor/golang.org/x/sys/unix/cap_freebsd.go | 195 + vendor/golang.org/x/sys/unix/constants.go | 13 + vendor/golang.org/x/sys/unix/creds_test.go | 136 + vendor/golang.org/x/sys/unix/dev_darwin.go | 24 + .../golang.org/x/sys/unix/dev_darwin_test.go | 49 + vendor/golang.org/x/sys/unix/dev_dragonfly.go | 30 + .../x/sys/unix/dev_dragonfly_test.go | 48 + vendor/golang.org/x/sys/unix/dev_freebsd.go | 30 + vendor/golang.org/x/sys/unix/dev_linux.go | 42 + .../golang.org/x/sys/unix/dev_linux_test.go | 51 + vendor/golang.org/x/sys/unix/dev_netbsd.go | 29 + .../golang.org/x/sys/unix/dev_netbsd_test.go | 50 + vendor/golang.org/x/sys/unix/dev_openbsd.go | 29 + .../golang.org/x/sys/unix/dev_openbsd_test.go | 52 + vendor/golang.org/x/sys/unix/dirent.go | 102 + vendor/golang.org/x/sys/unix/endian_big.go | 9 + vendor/golang.org/x/sys/unix/endian_little.go | 9 + vendor/golang.org/x/sys/unix/env_unix.go | 27 + vendor/golang.org/x/sys/unix/env_unset.go | 14 + .../x/sys/unix/errors_freebsd_386.go | 227 + .../x/sys/unix/errors_freebsd_amd64.go | 227 + .../x/sys/unix/errors_freebsd_arm.go | 226 + vendor/golang.org/x/sys/unix/export_test.go | 9 + vendor/golang.org/x/sys/unix/file_unix.go | 27 + vendor/golang.org/x/sys/unix/flock.go | 22 + .../x/sys/unix/flock_linux_32bit.go | 13 + vendor/golang.org/x/sys/unix/gccgo.go | 46 + vendor/golang.org/x/sys/unix/gccgo_c.c | 41 + .../x/sys/unix/gccgo_linux_amd64.go | 20 + vendor/golang.org/x/sys/unix/linux/Dockerfile | 51 + vendor/golang.org/x/sys/unix/linux/mkall.go | 379 + .../golang.org/x/sys/unix/linux/mksysnum.pl | 85 + vendor/golang.org/x/sys/unix/linux/types.go | 595 + vendor/golang.org/x/sys/unix/mkpost.go | 88 + vendor/golang.org/x/sys/unix/mksyscall.pl | 328 + .../x/sys/unix/mksyscall_solaris.pl | 289 + .../golang.org/x/sys/unix/mksysctl_openbsd.pl | 264 + .../golang.org/x/sys/unix/mksysnum_darwin.pl | 39 + .../x/sys/unix/mksysnum_dragonfly.pl | 50 + .../golang.org/x/sys/unix/mksysnum_freebsd.pl | 50 + .../golang.org/x/sys/unix/mksysnum_netbsd.pl | 58 + .../golang.org/x/sys/unix/mksysnum_openbsd.pl | 50 + .../golang.org/x/sys/unix/mmap_unix_test.go | 35 + .../golang.org/x/sys/unix/openbsd_pledge.go | 38 + vendor/golang.org/x/sys/unix/openbsd_test.go | 113 + vendor/golang.org/x/sys/unix/pagesize_unix.go | 15 + vendor/golang.org/x/sys/unix/race.go | 30 + vendor/golang.org/x/sys/unix/race0.go | 25 + .../golang.org/x/sys/unix/sockcmsg_linux.go | 36 + vendor/golang.org/x/sys/unix/sockcmsg_unix.go | 104 + vendor/golang.org/x/sys/unix/str.go | 26 + vendor/golang.org/x/sys/unix/syscall.go | 69 + vendor/golang.org/x/sys/unix/syscall_bsd.go | 635 + .../golang.org/x/sys/unix/syscall_bsd_test.go | 62 + .../golang.org/x/sys/unix/syscall_darwin.go | 536 + .../x/sys/unix/syscall_darwin_386.go | 75 + .../x/sys/unix/syscall_darwin_amd64.go | 75 + .../x/sys/unix/syscall_darwin_arm.go | 69 + .../x/sys/unix/syscall_darwin_arm64.go | 75 + .../x/sys/unix/syscall_dragonfly.go | 415 + .../x/sys/unix/syscall_dragonfly_amd64.go | 59 + .../golang.org/x/sys/unix/syscall_freebsd.go | 708 + .../x/sys/unix/syscall_freebsd_386.go | 59 + .../x/sys/unix/syscall_freebsd_amd64.go | 59 + .../x/sys/unix/syscall_freebsd_arm.go | 59 + .../x/sys/unix/syscall_freebsd_test.go | 297 + vendor/golang.org/x/sys/unix/syscall_linux.go | 1469 ++ .../x/sys/unix/syscall_linux_386.go | 397 + .../x/sys/unix/syscall_linux_amd64.go | 150 + .../x/sys/unix/syscall_linux_amd64_gc.go | 13 + .../x/sys/unix/syscall_linux_arm.go | 261 + .../x/sys/unix/syscall_linux_arm64.go | 188 + .../x/sys/unix/syscall_linux_mips64x.go | 207 + .../x/sys/unix/syscall_linux_mipsx.go | 237 + .../x/sys/unix/syscall_linux_ppc64x.go | 133 + .../x/sys/unix/syscall_linux_s390x.go | 326 + .../x/sys/unix/syscall_linux_sparc64.go | 149 + .../x/sys/unix/syscall_linux_test.go | 227 + .../golang.org/x/sys/unix/syscall_netbsd.go | 472 + .../x/sys/unix/syscall_netbsd_386.go | 40 + .../x/sys/unix/syscall_netbsd_amd64.go | 40 + .../x/sys/unix/syscall_netbsd_arm.go | 40 + .../golang.org/x/sys/unix/syscall_no_getwd.go | 11 + .../golang.org/x/sys/unix/syscall_openbsd.go | 282 + .../x/sys/unix/syscall_openbsd_386.go | 40 + .../x/sys/unix/syscall_openbsd_amd64.go | 40 + .../x/sys/unix/syscall_openbsd_arm.go | 40 + .../golang.org/x/sys/unix/syscall_solaris.go | 700 + .../x/sys/unix/syscall_solaris_amd64.go | 35 + .../x/sys/unix/syscall_solaris_test.go | 34 + vendor/golang.org/x/sys/unix/syscall_test.go | 50 + vendor/golang.org/x/sys/unix/syscall_unix.go | 293 + .../golang.org/x/sys/unix/syscall_unix_gc.go | 15 + .../x/sys/unix/syscall_unix_test.go | 345 + vendor/golang.org/x/sys/unix/types_darwin.go | 254 + .../golang.org/x/sys/unix/types_dragonfly.go | 249 + vendor/golang.org/x/sys/unix/types_freebsd.go | 372 + vendor/golang.org/x/sys/unix/types_netbsd.go | 239 + vendor/golang.org/x/sys/unix/types_openbsd.go | 251 + vendor/golang.org/x/sys/unix/types_solaris.go | 265 + .../x/sys/unix/zerrors_darwin_386.go | 1673 ++ .../x/sys/unix/zerrors_darwin_amd64.go | 1673 ++ .../x/sys/unix/zerrors_darwin_arm.go | 1673 ++ .../x/sys/unix/zerrors_darwin_arm64.go | 1673 ++ .../x/sys/unix/zerrors_dragonfly_amd64.go | 1568 ++ .../x/sys/unix/zerrors_freebsd_386.go | 1706 ++ .../x/sys/unix/zerrors_freebsd_amd64.go | 1707 ++ .../x/sys/unix/zerrors_freebsd_arm.go | 1715 ++ .../x/sys/unix/zerrors_linux_386.go | 2231 ++ .../x/sys/unix/zerrors_linux_amd64.go | 2232 ++ .../x/sys/unix/zerrors_linux_arm.go | 2236 +++ .../x/sys/unix/zerrors_linux_arm64.go | 2222 ++ .../x/sys/unix/zerrors_linux_mips.go | 2241 +++ .../x/sys/unix/zerrors_linux_mips64.go | 2241 +++ .../x/sys/unix/zerrors_linux_mips64le.go | 2241 +++ .../x/sys/unix/zerrors_linux_mipsle.go | 2241 +++ .../x/sys/unix/zerrors_linux_ppc64.go | 2294 +++ .../x/sys/unix/zerrors_linux_ppc64le.go | 2294 +++ .../x/sys/unix/zerrors_linux_s390x.go | 2293 +++ .../x/sys/unix/zerrors_linux_sparc64.go | 2142 ++ .../x/sys/unix/zerrors_netbsd_386.go | 1712 ++ .../x/sys/unix/zerrors_netbsd_amd64.go | 1702 ++ .../x/sys/unix/zerrors_netbsd_arm.go | 1691 ++ .../x/sys/unix/zerrors_openbsd_386.go | 1584 ++ .../x/sys/unix/zerrors_openbsd_amd64.go | 1583 ++ .../x/sys/unix/zerrors_openbsd_arm.go | 1586 ++ .../x/sys/unix/zerrors_solaris_amd64.go | 1483 ++ .../x/sys/unix/zsyscall_darwin_386.go | 1609 ++ .../x/sys/unix/zsyscall_darwin_amd64.go | 1609 ++ .../x/sys/unix/zsyscall_darwin_arm.go | 1609 ++ .../x/sys/unix/zsyscall_darwin_arm64.go | 1609 ++ .../x/sys/unix/zsyscall_dragonfly_amd64.go | 1440 ++ .../x/sys/unix/zsyscall_freebsd_386.go | 1877 ++ .../x/sys/unix/zsyscall_freebsd_amd64.go | 1877 ++ .../x/sys/unix/zsyscall_freebsd_arm.go | 1877 ++ .../x/sys/unix/zsyscall_linux_386.go | 1953 ++ .../x/sys/unix/zsyscall_linux_amd64.go | 2146 ++ .../x/sys/unix/zsyscall_linux_arm.go | 2055 ++ .../x/sys/unix/zsyscall_linux_arm64.go | 2029 ++ .../x/sys/unix/zsyscall_linux_mips.go | 2111 ++ .../x/sys/unix/zsyscall_linux_mips64.go | 2105 ++ .../x/sys/unix/zsyscall_linux_mips64le.go | 2105 ++ .../x/sys/unix/zsyscall_linux_mipsle.go | 2111 ++ .../x/sys/unix/zsyscall_linux_ppc64.go | 2157 ++ .../x/sys/unix/zsyscall_linux_ppc64le.go | 2157 ++ .../x/sys/unix/zsyscall_linux_s390x.go | 1937 ++ .../x/sys/unix/zsyscall_linux_sparc64.go | 1833 ++ .../x/sys/unix/zsyscall_netbsd_386.go | 1346 ++ .../x/sys/unix/zsyscall_netbsd_amd64.go | 1346 ++ .../x/sys/unix/zsyscall_netbsd_arm.go | 1346 ++ .../x/sys/unix/zsyscall_openbsd_386.go | 1404 ++ .../x/sys/unix/zsyscall_openbsd_amd64.go | 1404 ++ .../x/sys/unix/zsyscall_openbsd_arm.go | 1404 ++ .../x/sys/unix/zsyscall_solaris_amd64.go | 1588 ++ .../golang.org/x/sys/unix/zsysctl_openbsd.go | 270 + .../x/sys/unix/zsysnum_darwin_386.go | 398 + .../x/sys/unix/zsysnum_darwin_amd64.go | 398 + .../x/sys/unix/zsysnum_darwin_arm.go | 426 + .../x/sys/unix/zsysnum_darwin_arm64.go | 426 + .../x/sys/unix/zsysnum_dragonfly_amd64.go | 315 + .../x/sys/unix/zsysnum_freebsd_386.go | 353 + .../x/sys/unix/zsysnum_freebsd_amd64.go | 353 + .../x/sys/unix/zsysnum_freebsd_arm.go | 353 + .../x/sys/unix/zsysnum_linux_386.go | 390 + .../x/sys/unix/zsysnum_linux_amd64.go | 342 + .../x/sys/unix/zsysnum_linux_arm.go | 362 + .../x/sys/unix/zsysnum_linux_arm64.go | 286 + .../x/sys/unix/zsysnum_linux_mips.go | 375 + .../x/sys/unix/zsysnum_linux_mips64.go | 335 + .../x/sys/unix/zsysnum_linux_mips64le.go | 335 + .../x/sys/unix/zsysnum_linux_mipsle.go | 375 + .../x/sys/unix/zsysnum_linux_ppc64.go | 370 + .../x/sys/unix/zsysnum_linux_ppc64le.go | 370 + .../x/sys/unix/zsysnum_linux_s390x.go | 333 + .../x/sys/unix/zsysnum_linux_sparc64.go | 348 + .../x/sys/unix/zsysnum_netbsd_386.go | 274 + .../x/sys/unix/zsysnum_netbsd_amd64.go | 274 + .../x/sys/unix/zsysnum_netbsd_arm.go | 274 + .../x/sys/unix/zsysnum_openbsd_386.go | 207 + .../x/sys/unix/zsysnum_openbsd_amd64.go | 207 + .../x/sys/unix/zsysnum_openbsd_arm.go | 213 + .../x/sys/unix/zsysnum_solaris_amd64.go | 13 + .../x/sys/unix/ztypes_darwin_386.go | 462 + .../x/sys/unix/ztypes_darwin_amd64.go | 472 + .../x/sys/unix/ztypes_darwin_arm.go | 463 + .../x/sys/unix/ztypes_darwin_arm64.go | 471 + .../x/sys/unix/ztypes_dragonfly_amd64.go | 448 + .../x/sys/unix/ztypes_freebsd_386.go | 521 + .../x/sys/unix/ztypes_freebsd_amd64.go | 524 + .../x/sys/unix/ztypes_freebsd_arm.go | 524 + .../golang.org/x/sys/unix/ztypes_linux_386.go | 793 + .../x/sys/unix/ztypes_linux_amd64.go | 811 + .../golang.org/x/sys/unix/ztypes_linux_arm.go | 782 + .../x/sys/unix/ztypes_linux_arm64.go | 790 + .../x/sys/unix/ztypes_linux_mips.go | 787 + .../x/sys/unix/ztypes_linux_mips64.go | 792 + .../x/sys/unix/ztypes_linux_mips64le.go | 792 + .../x/sys/unix/ztypes_linux_mipsle.go | 787 + .../x/sys/unix/ztypes_linux_ppc64.go | 800 + .../x/sys/unix/ztypes_linux_ppc64le.go | 800 + .../x/sys/unix/ztypes_linux_s390x.go | 817 + .../x/sys/unix/ztypes_linux_sparc64.go | 666 + .../x/sys/unix/ztypes_netbsd_386.go | 401 + .../x/sys/unix/ztypes_netbsd_amd64.go | 408 + .../x/sys/unix/ztypes_netbsd_arm.go | 406 + .../x/sys/unix/ztypes_openbsd_386.go | 446 + .../x/sys/unix/ztypes_openbsd_amd64.go | 453 + .../x/sys/unix/ztypes_openbsd_arm.go | 439 + .../x/sys/unix/ztypes_solaris_amd64.go | 442 + .../x/sys/windows/asm_windows_386.s | 13 + .../x/sys/windows/asm_windows_amd64.s | 13 + .../golang.org/x/sys/windows/dll_windows.go | 377 + vendor/golang.org/x/sys/windows/env_unset.go | 15 + .../golang.org/x/sys/windows/env_windows.go | 25 + vendor/golang.org/x/sys/windows/eventlog.go | 20 + .../golang.org/x/sys/windows/exec_windows.go | 97 + .../x/sys/windows/memory_windows.go | 26 + vendor/golang.org/x/sys/windows/mksyscall.go | 7 + vendor/golang.org/x/sys/windows/race.go | 30 + vendor/golang.org/x/sys/windows/race0.go | 25 + .../x/sys/windows/registry/export_test.go | 11 + .../golang.org/x/sys/windows/registry/key.go | 200 + .../x/sys/windows/registry/mksyscall.go | 7 + .../x/sys/windows/registry/registry_test.go | 756 + .../x/sys/windows/registry/syscall.go | 32 + .../x/sys/windows/registry/value.go | 384 + .../sys/windows/registry/zsyscall_windows.go | 120 + .../x/sys/windows/security_windows.go | 435 + vendor/golang.org/x/sys/windows/service.go | 164 + vendor/golang.org/x/sys/windows/str.go | 22 + .../golang.org/x/sys/windows/svc/debug/log.go | 56 + .../x/sys/windows/svc/debug/service.go | 45 + vendor/golang.org/x/sys/windows/svc/event.go | 48 + .../x/sys/windows/svc/eventlog/install.go | 80 + .../x/sys/windows/svc/eventlog/log.go | 70 + .../x/sys/windows/svc/eventlog/log_test.go | 51 + .../x/sys/windows/svc/example/beep.go | 22 + .../x/sys/windows/svc/example/install.go | 92 + .../x/sys/windows/svc/example/main.go | 76 + .../x/sys/windows/svc/example/manage.go | 62 + .../x/sys/windows/svc/example/service.go | 82 + vendor/golang.org/x/sys/windows/svc/go12.c | 24 + vendor/golang.org/x/sys/windows/svc/go12.go | 11 + vendor/golang.org/x/sys/windows/svc/go13.go | 31 + .../x/sys/windows/svc/mgr/config.go | 139 + .../golang.org/x/sys/windows/svc/mgr/mgr.go | 162 + .../x/sys/windows/svc/mgr/mgr_test.go | 169 + .../x/sys/windows/svc/mgr/service.go | 72 + .../golang.org/x/sys/windows/svc/security.go | 62 + .../golang.org/x/sys/windows/svc/service.go | 363 + .../golang.org/x/sys/windows/svc/svc_test.go | 118 + vendor/golang.org/x/sys/windows/svc/sys_386.s | 68 + .../golang.org/x/sys/windows/svc/sys_amd64.s | 42 + vendor/golang.org/x/sys/windows/syscall.go | 71 + .../golang.org/x/sys/windows/syscall_test.go | 33 + .../x/sys/windows/syscall_windows.go | 1009 + .../x/sys/windows/syscall_windows_test.go | 107 + .../golang.org/x/sys/windows/types_windows.go | 1282 ++ .../x/sys/windows/types_windows_386.go | 22 + .../x/sys/windows/types_windows_amd64.go | 22 + .../x/sys/windows/zsyscall_windows.go | 2428 +++ 1138 files changed, 354771 insertions(+) create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 Gopkg.lock create mode 100644 Gopkg.toml create mode 100644 LICENSE create mode 100644 README.md create mode 100644 sage.default.conf create mode 100644 sage.go create mode 100644 vendor/github.com/BurntSushi/toml/.gitignore create mode 100644 vendor/github.com/BurntSushi/toml/.travis.yml create mode 100644 vendor/github.com/BurntSushi/toml/COMPATIBLE create mode 100644 vendor/github.com/BurntSushi/toml/COPYING create mode 100644 vendor/github.com/BurntSushi/toml/Makefile create mode 100644 vendor/github.com/BurntSushi/toml/README.md create mode 100644 vendor/github.com/BurntSushi/toml/_examples/example.go create mode 100644 vendor/github.com/BurntSushi/toml/_examples/example.toml create mode 100644 vendor/github.com/BurntSushi/toml/_examples/hard.toml create mode 100644 vendor/github.com/BurntSushi/toml/_examples/implicit.toml create mode 100644 vendor/github.com/BurntSushi/toml/_examples/invalid-apples.toml create mode 100644 vendor/github.com/BurntSushi/toml/_examples/invalid.toml create mode 100644 vendor/github.com/BurntSushi/toml/_examples/readme1.toml create mode 100644 vendor/github.com/BurntSushi/toml/_examples/readme2.toml create mode 100644 vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING create mode 100644 vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/README.md create mode 100644 vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/main.go create mode 100644 vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING create mode 100644 vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/README.md create mode 100644 vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/main.go create mode 100644 vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING create mode 100644 vendor/github.com/BurntSushi/toml/cmd/tomlv/README.md create mode 100644 vendor/github.com/BurntSushi/toml/cmd/tomlv/main.go create mode 100644 vendor/github.com/BurntSushi/toml/decode.go create mode 100644 vendor/github.com/BurntSushi/toml/decode_meta.go create mode 100644 vendor/github.com/BurntSushi/toml/decode_test.go create mode 100644 vendor/github.com/BurntSushi/toml/doc.go create mode 100644 vendor/github.com/BurntSushi/toml/encode.go create mode 100644 vendor/github.com/BurntSushi/toml/encode_test.go create mode 100644 vendor/github.com/BurntSushi/toml/encoding_types.go create mode 100644 vendor/github.com/BurntSushi/toml/encoding_types_1.1.go create mode 100644 vendor/github.com/BurntSushi/toml/lex.go create mode 100644 vendor/github.com/BurntSushi/toml/parse.go create mode 100644 vendor/github.com/BurntSushi/toml/session.vim create mode 100644 vendor/github.com/BurntSushi/toml/type_check.go create mode 100644 vendor/github.com/BurntSushi/toml/type_fields.go create mode 100644 vendor/github.com/coreos/bbolt/.gitignore create mode 100644 vendor/github.com/coreos/bbolt/LICENSE create mode 100644 vendor/github.com/coreos/bbolt/Makefile create mode 100644 vendor/github.com/coreos/bbolt/README.md create mode 100644 vendor/github.com/coreos/bbolt/appveyor.yml create mode 100644 vendor/github.com/coreos/bbolt/bolt_386.go create mode 100644 vendor/github.com/coreos/bbolt/bolt_amd64.go create mode 100644 vendor/github.com/coreos/bbolt/bolt_arm.go create mode 100644 vendor/github.com/coreos/bbolt/bolt_arm64.go create mode 100644 vendor/github.com/coreos/bbolt/bolt_linux.go create mode 100644 vendor/github.com/coreos/bbolt/bolt_openbsd.go create mode 100644 vendor/github.com/coreos/bbolt/bolt_ppc.go create mode 100644 vendor/github.com/coreos/bbolt/bolt_ppc64.go create mode 100644 vendor/github.com/coreos/bbolt/bolt_ppc64le.go create mode 100644 vendor/github.com/coreos/bbolt/bolt_s390x.go create mode 100644 vendor/github.com/coreos/bbolt/bolt_unix.go create mode 100644 vendor/github.com/coreos/bbolt/bolt_unix_solaris.go create mode 100644 vendor/github.com/coreos/bbolt/bolt_windows.go create mode 100644 vendor/github.com/coreos/bbolt/boltsync_unix.go create mode 100644 vendor/github.com/coreos/bbolt/bucket.go create mode 100644 vendor/github.com/coreos/bbolt/bucket_test.go create mode 100644 vendor/github.com/coreos/bbolt/cmd/bolt/main.go create mode 100644 vendor/github.com/coreos/bbolt/cmd/bolt/main_test.go create mode 100644 vendor/github.com/coreos/bbolt/cursor.go create mode 100644 vendor/github.com/coreos/bbolt/cursor_test.go create mode 100644 vendor/github.com/coreos/bbolt/db.go create mode 100644 vendor/github.com/coreos/bbolt/db_test.go create mode 100644 vendor/github.com/coreos/bbolt/doc.go create mode 100644 vendor/github.com/coreos/bbolt/errors.go create mode 100644 vendor/github.com/coreos/bbolt/freelist.go create mode 100644 vendor/github.com/coreos/bbolt/freelist_test.go create mode 100644 vendor/github.com/coreos/bbolt/node.go create mode 100644 vendor/github.com/coreos/bbolt/node_test.go create mode 100644 vendor/github.com/coreos/bbolt/page.go create mode 100644 vendor/github.com/coreos/bbolt/page_test.go create mode 100644 vendor/github.com/coreos/bbolt/quick_test.go create mode 100644 vendor/github.com/coreos/bbolt/simulation_test.go create mode 100644 vendor/github.com/coreos/bbolt/tx.go create mode 100644 vendor/github.com/coreos/bbolt/tx_test.go create mode 100644 vendor/github.com/dustin/go-humanize/.gitignore create mode 100644 vendor/github.com/dustin/go-humanize/.travis.yml create mode 100644 vendor/github.com/dustin/go-humanize/LICENSE create mode 100644 vendor/github.com/dustin/go-humanize/README.markdown create mode 100644 vendor/github.com/dustin/go-humanize/big.go create mode 100644 vendor/github.com/dustin/go-humanize/bigbytes.go create mode 100644 vendor/github.com/dustin/go-humanize/bigbytes_test.go create mode 100644 vendor/github.com/dustin/go-humanize/bytes.go create mode 100644 vendor/github.com/dustin/go-humanize/bytes_test.go create mode 100644 vendor/github.com/dustin/go-humanize/comma.go create mode 100644 vendor/github.com/dustin/go-humanize/comma_test.go create mode 100644 vendor/github.com/dustin/go-humanize/commaf.go create mode 100644 vendor/github.com/dustin/go-humanize/commaf_test.go create mode 100644 vendor/github.com/dustin/go-humanize/common_test.go create mode 100644 vendor/github.com/dustin/go-humanize/ftoa.go create mode 100644 vendor/github.com/dustin/go-humanize/ftoa_test.go create mode 100644 vendor/github.com/dustin/go-humanize/humanize.go create mode 100644 vendor/github.com/dustin/go-humanize/number.go create mode 100644 vendor/github.com/dustin/go-humanize/number_test.go create mode 100644 vendor/github.com/dustin/go-humanize/ordinals.go create mode 100644 vendor/github.com/dustin/go-humanize/ordinals_test.go create mode 100644 vendor/github.com/dustin/go-humanize/si.go create mode 100644 vendor/github.com/dustin/go-humanize/si_test.go create mode 100644 vendor/github.com/dustin/go-humanize/times.go create mode 100644 vendor/github.com/dustin/go-humanize/times_test.go create mode 100644 vendor/github.com/fluffle/goirc/.gitignore create mode 100644 vendor/github.com/fluffle/goirc/.travis.yml create mode 100644 vendor/github.com/fluffle/goirc/LICENSE create mode 100644 vendor/github.com/fluffle/goirc/README.md create mode 100644 vendor/github.com/fluffle/goirc/client.go create mode 100644 vendor/github.com/fluffle/goirc/client/commands.go create mode 100644 vendor/github.com/fluffle/goirc/client/commands_test.go create mode 100644 vendor/github.com/fluffle/goirc/client/connection.go create mode 100644 vendor/github.com/fluffle/goirc/client/connection_test.go create mode 100644 vendor/github.com/fluffle/goirc/client/dispatch.go create mode 100644 vendor/github.com/fluffle/goirc/client/dispatch_test.go create mode 100644 vendor/github.com/fluffle/goirc/client/doc.go create mode 100644 vendor/github.com/fluffle/goirc/client/handlers.go create mode 100644 vendor/github.com/fluffle/goirc/client/handlers_test.go create mode 100644 vendor/github.com/fluffle/goirc/client/line.go create mode 100644 vendor/github.com/fluffle/goirc/client/line_test.go create mode 100644 vendor/github.com/fluffle/goirc/client/mocknetconn_test.go create mode 100644 vendor/github.com/fluffle/goirc/client/state_handlers.go create mode 100644 vendor/github.com/fluffle/goirc/doc/rfc2811.txt create mode 100644 vendor/github.com/fluffle/goirc/doc/rfc2812.txt create mode 100644 vendor/github.com/fluffle/goirc/doc/unreal32docs.html create mode 100644 vendor/github.com/fluffle/goirc/logging/glog/glog.go create mode 100644 vendor/github.com/fluffle/goirc/logging/golog/golog.go create mode 100644 vendor/github.com/fluffle/goirc/logging/logging.go create mode 100644 vendor/github.com/fluffle/goirc/state/channel.go create mode 100644 vendor/github.com/fluffle/goirc/state/channel_test.go create mode 100644 vendor/github.com/fluffle/goirc/state/mock_tracker.go create mode 100644 vendor/github.com/fluffle/goirc/state/nick.go create mode 100644 vendor/github.com/fluffle/goirc/state/nick_test.go create mode 100644 vendor/github.com/fluffle/goirc/state/tracker.go create mode 100644 vendor/github.com/fluffle/goirc/state/tracker_test.go create mode 100644 vendor/github.com/fluffle/goirc/vims create mode 100644 vendor/github.com/golang/mock/.gitignore create mode 100644 vendor/github.com/golang/mock/.travis.yml create mode 100644 vendor/github.com/golang/mock/AUTHORS create mode 100644 vendor/github.com/golang/mock/CONTRIBUTORS create mode 100644 vendor/github.com/golang/mock/LICENSE create mode 100644 vendor/github.com/golang/mock/README.md create mode 100644 vendor/github.com/golang/mock/gomock/call.go create mode 100644 vendor/github.com/golang/mock/gomock/call_test.go create mode 100644 vendor/github.com/golang/mock/gomock/callset.go create mode 100644 vendor/github.com/golang/mock/gomock/controller.go create mode 100644 vendor/github.com/golang/mock/gomock/controller_test.go create mode 100644 vendor/github.com/golang/mock/gomock/matchers.go create mode 100644 vendor/github.com/golang/mock/gomock/matchers_test.go create mode 100644 vendor/github.com/golang/mock/gomock/mock_matcher/mock_matcher.go create mode 100644 vendor/github.com/golang/mock/mockgen/mockgen.go create mode 100644 vendor/github.com/golang/mock/mockgen/model/model.go create mode 100644 vendor/github.com/golang/mock/mockgen/parse.go create mode 100644 vendor/github.com/golang/mock/mockgen/reflect.go create mode 100644 vendor/github.com/golang/mock/mockgen/tests/unexported_method/README.md create mode 100644 vendor/github.com/golang/mock/mockgen/tests/unexported_method/bugreport.go create mode 100644 vendor/github.com/golang/mock/mockgen/tests/unexported_method/bugreport_mock.go create mode 100644 vendor/github.com/golang/mock/mockgen/tests/unexported_method/bugreport_test.go create mode 100644 vendor/github.com/golang/mock/sample/README.md create mode 100644 vendor/github.com/golang/mock/sample/imp1/imp1.go create mode 100644 vendor/github.com/golang/mock/sample/imp2/imp2.go create mode 100644 vendor/github.com/golang/mock/sample/imp3/imp3.go create mode 100644 vendor/github.com/golang/mock/sample/imp4/imp4.go create mode 100644 vendor/github.com/golang/mock/sample/mock_user/mock_user.go create mode 100644 vendor/github.com/golang/mock/sample/user.go create mode 100644 vendor/github.com/golang/mock/sample/user_test.go create mode 100644 vendor/github.com/mandykoh/keva/.gitignore create mode 100644 vendor/github.com/mandykoh/keva/.travis.yml create mode 100644 vendor/github.com/mandykoh/keva/LICENSE create mode 100644 vendor/github.com/mandykoh/keva/README.md create mode 100644 vendor/github.com/mandykoh/keva/bucket.go create mode 100644 vendor/github.com/mandykoh/keva/bucket_test.go create mode 100644 vendor/github.com/mandykoh/keva/bucketcache.go create mode 100644 vendor/github.com/mandykoh/keva/bucketcache_test.go create mode 100644 vendor/github.com/mandykoh/keva/bucketcacheentry.go create mode 100644 vendor/github.com/mandykoh/keva/bucketcachetrie.go create mode 100644 vendor/github.com/mandykoh/keva/bucketcachetrie_test.go create mode 100644 vendor/github.com/mandykoh/keva/bucketpath.go create mode 100644 vendor/github.com/mandykoh/keva/bucketpath_test.go create mode 100644 vendor/github.com/mandykoh/keva/common_test.go create mode 100644 vendor/github.com/mandykoh/keva/store.go create mode 100644 vendor/github.com/mandykoh/keva/store_test.go create mode 100644 vendor/github.com/mandykoh/keva/storeinfo.go create mode 100644 vendor/github.com/mandykoh/symlock/.travis.yml create mode 100644 vendor/github.com/mandykoh/symlock/LICENSE create mode 100644 vendor/github.com/mandykoh/symlock/README.md create mode 100644 vendor/github.com/mandykoh/symlock/partition.go create mode 100644 vendor/github.com/mandykoh/symlock/symlock.go create mode 100644 vendor/github.com/mandykoh/symlock/symlock_test.go create mode 100644 vendor/github.com/mandykoh/symlock/symlockentry.go create mode 100644 vendor/github.com/tslocum/markov/.gitignore create mode 100644 vendor/github.com/tslocum/markov/.travis.yml create mode 100644 vendor/github.com/tslocum/markov/LICENSE create mode 100644 vendor/github.com/tslocum/markov/README.md create mode 100644 vendor/github.com/tslocum/markov/accumulator.go create mode 100644 vendor/github.com/tslocum/markov/accumulator_test.go create mode 100644 vendor/github.com/tslocum/markov/bolttablestore.go create mode 100644 vendor/github.com/tslocum/markov/generator.go create mode 100644 vendor/github.com/tslocum/markov/generator_test.go create mode 100644 vendor/github.com/tslocum/markov/inmemorytablestore.go create mode 100644 vendor/github.com/tslocum/markov/kevatablestore.go create mode 100644 vendor/github.com/tslocum/markov/model.go create mode 100644 vendor/github.com/tslocum/markov/model_test.go create mode 100644 vendor/github.com/tslocum/markov/sequence.go create mode 100644 vendor/github.com/tslocum/markov/sequence_test.go create mode 100644 vendor/github.com/tslocum/markov/table.go create mode 100644 vendor/github.com/tslocum/markov/table_test.go create mode 100644 vendor/github.com/tslocum/markov/tableentry.go create mode 100644 vendor/github.com/tslocum/markov/tablestore.go create mode 100644 vendor/golang.org/x/net/.gitattributes create mode 100644 vendor/golang.org/x/net/.gitignore create mode 100644 vendor/golang.org/x/net/AUTHORS create mode 100644 vendor/golang.org/x/net/CONTRIBUTING.md create mode 100644 vendor/golang.org/x/net/CONTRIBUTORS create mode 100644 vendor/golang.org/x/net/LICENSE create mode 100644 vendor/golang.org/x/net/PATENTS create mode 100644 vendor/golang.org/x/net/README create mode 100644 vendor/golang.org/x/net/bpf/asm.go create mode 100644 vendor/golang.org/x/net/bpf/constants.go create mode 100644 vendor/golang.org/x/net/bpf/doc.go create mode 100644 vendor/golang.org/x/net/bpf/instructions.go create mode 100644 vendor/golang.org/x/net/bpf/instructions_test.go create mode 100644 vendor/golang.org/x/net/bpf/setter.go create mode 100644 vendor/golang.org/x/net/bpf/testdata/all_instructions.bpf create mode 100644 vendor/golang.org/x/net/bpf/testdata/all_instructions.txt create mode 100644 vendor/golang.org/x/net/bpf/vm.go create mode 100644 vendor/golang.org/x/net/bpf/vm_aluop_test.go create mode 100644 vendor/golang.org/x/net/bpf/vm_bpf_test.go create mode 100644 vendor/golang.org/x/net/bpf/vm_extension_test.go create mode 100644 vendor/golang.org/x/net/bpf/vm_instructions.go create mode 100644 vendor/golang.org/x/net/bpf/vm_jump_test.go create mode 100644 vendor/golang.org/x/net/bpf/vm_load_test.go create mode 100644 vendor/golang.org/x/net/bpf/vm_ret_test.go create mode 100644 vendor/golang.org/x/net/bpf/vm_scratch_test.go create mode 100644 vendor/golang.org/x/net/bpf/vm_test.go create mode 100644 vendor/golang.org/x/net/codereview.cfg create mode 100644 vendor/golang.org/x/net/context/context.go create mode 100644 vendor/golang.org/x/net/context/context_test.go create mode 100644 vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go create mode 100644 vendor/golang.org/x/net/context/ctxhttp/ctxhttp_17_test.go create mode 100644 vendor/golang.org/x/net/context/ctxhttp/ctxhttp_pre17.go create mode 100644 vendor/golang.org/x/net/context/ctxhttp/ctxhttp_pre17_test.go create mode 100644 vendor/golang.org/x/net/context/ctxhttp/ctxhttp_test.go create mode 100644 vendor/golang.org/x/net/context/go17.go create mode 100644 vendor/golang.org/x/net/context/go19.go create mode 100644 vendor/golang.org/x/net/context/pre_go17.go create mode 100644 vendor/golang.org/x/net/context/pre_go19.go create mode 100644 vendor/golang.org/x/net/context/withtimeout_test.go create mode 100644 vendor/golang.org/x/net/dict/dict.go create mode 100644 vendor/golang.org/x/net/dns/dnsmessage/example_test.go create mode 100644 vendor/golang.org/x/net/dns/dnsmessage/message.go create mode 100644 vendor/golang.org/x/net/dns/dnsmessage/message_test.go create mode 100644 vendor/golang.org/x/net/html/atom/atom.go create mode 100644 vendor/golang.org/x/net/html/atom/atom_test.go create mode 100644 vendor/golang.org/x/net/html/atom/gen.go create mode 100644 vendor/golang.org/x/net/html/atom/table.go create mode 100644 vendor/golang.org/x/net/html/atom/table_test.go create mode 100644 vendor/golang.org/x/net/html/charset/charset.go create mode 100644 vendor/golang.org/x/net/html/charset/charset_test.go create mode 100644 vendor/golang.org/x/net/html/charset/testdata/HTTP-charset.html create mode 100644 vendor/golang.org/x/net/html/charset/testdata/HTTP-vs-UTF-8-BOM.html create mode 100644 vendor/golang.org/x/net/html/charset/testdata/HTTP-vs-meta-charset.html create mode 100644 vendor/golang.org/x/net/html/charset/testdata/HTTP-vs-meta-content.html create mode 100644 vendor/golang.org/x/net/html/charset/testdata/No-encoding-declaration.html create mode 100644 vendor/golang.org/x/net/html/charset/testdata/README create mode 100644 vendor/golang.org/x/net/html/charset/testdata/UTF-16BE-BOM.html create mode 100644 vendor/golang.org/x/net/html/charset/testdata/UTF-16LE-BOM.html create mode 100644 vendor/golang.org/x/net/html/charset/testdata/UTF-8-BOM-vs-meta-charset.html create mode 100644 vendor/golang.org/x/net/html/charset/testdata/UTF-8-BOM-vs-meta-content.html create mode 100644 vendor/golang.org/x/net/html/charset/testdata/meta-charset-attribute.html create mode 100644 vendor/golang.org/x/net/html/charset/testdata/meta-content-attribute.html create mode 100644 vendor/golang.org/x/net/html/const.go create mode 100644 vendor/golang.org/x/net/html/doc.go create mode 100644 vendor/golang.org/x/net/html/doctype.go create mode 100644 vendor/golang.org/x/net/html/entity.go create mode 100644 vendor/golang.org/x/net/html/entity_test.go create mode 100644 vendor/golang.org/x/net/html/escape.go create mode 100644 vendor/golang.org/x/net/html/escape_test.go create mode 100644 vendor/golang.org/x/net/html/example_test.go create mode 100644 vendor/golang.org/x/net/html/foreign.go create mode 100644 vendor/golang.org/x/net/html/node.go create mode 100644 vendor/golang.org/x/net/html/node_test.go create mode 100644 vendor/golang.org/x/net/html/parse.go create mode 100644 vendor/golang.org/x/net/html/parse_test.go create mode 100644 vendor/golang.org/x/net/html/render.go create mode 100644 vendor/golang.org/x/net/html/render_test.go create mode 100644 vendor/golang.org/x/net/html/testdata/go1.html create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/README create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/adoption01.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/adoption02.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/comments01.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/doctype01.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/entities01.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/entities02.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/html5test-com.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/inbody01.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/isindex.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/pending-spec-changes-plain-text-unsafe.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/pending-spec-changes.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/plain-text-unsafe.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/scriptdata01.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/scripted/adoption01.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/scripted/webkit01.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tables01.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests1.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests10.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests11.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests12.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests14.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests15.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests16.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests17.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests18.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests19.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests2.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests20.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests21.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests22.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests23.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests24.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests25.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests26.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests3.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests4.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests5.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests6.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests7.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests8.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests9.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tests_innerHTML_1.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/tricky01.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/webkit01.dat create mode 100644 vendor/golang.org/x/net/html/testdata/webkit/webkit02.dat create mode 100644 vendor/golang.org/x/net/html/token.go create mode 100644 vendor/golang.org/x/net/html/token_test.go create mode 100644 vendor/golang.org/x/net/http2/.gitignore create mode 100644 vendor/golang.org/x/net/http2/Dockerfile create mode 100644 vendor/golang.org/x/net/http2/Makefile create mode 100644 vendor/golang.org/x/net/http2/README create mode 100644 vendor/golang.org/x/net/http2/ciphers.go create mode 100644 vendor/golang.org/x/net/http2/ciphers_test.go create mode 100644 vendor/golang.org/x/net/http2/client_conn_pool.go create mode 100644 vendor/golang.org/x/net/http2/configure_transport.go create mode 100644 vendor/golang.org/x/net/http2/databuffer.go create mode 100644 vendor/golang.org/x/net/http2/databuffer_test.go create mode 100644 vendor/golang.org/x/net/http2/errors.go create mode 100644 vendor/golang.org/x/net/http2/errors_test.go create mode 100644 vendor/golang.org/x/net/http2/flow.go create mode 100644 vendor/golang.org/x/net/http2/flow_test.go create mode 100644 vendor/golang.org/x/net/http2/frame.go create mode 100644 vendor/golang.org/x/net/http2/frame_test.go create mode 100644 vendor/golang.org/x/net/http2/go16.go create mode 100644 vendor/golang.org/x/net/http2/go17.go create mode 100644 vendor/golang.org/x/net/http2/go17_not18.go create mode 100644 vendor/golang.org/x/net/http2/go18.go create mode 100644 vendor/golang.org/x/net/http2/go18_test.go create mode 100644 vendor/golang.org/x/net/http2/go19.go create mode 100644 vendor/golang.org/x/net/http2/go19_test.go create mode 100644 vendor/golang.org/x/net/http2/gotrack.go create mode 100644 vendor/golang.org/x/net/http2/gotrack_test.go create mode 100644 vendor/golang.org/x/net/http2/h2demo/.gitignore create mode 100644 vendor/golang.org/x/net/http2/h2demo/Makefile create mode 100644 vendor/golang.org/x/net/http2/h2demo/README create mode 100644 vendor/golang.org/x/net/http2/h2demo/h2demo.go create mode 100644 vendor/golang.org/x/net/http2/h2demo/launch.go create mode 100644 vendor/golang.org/x/net/http2/h2demo/rootCA.key create mode 100644 vendor/golang.org/x/net/http2/h2demo/rootCA.pem create mode 100644 vendor/golang.org/x/net/http2/h2demo/rootCA.srl create mode 100644 vendor/golang.org/x/net/http2/h2demo/server.crt create mode 100644 vendor/golang.org/x/net/http2/h2demo/server.key create mode 100644 vendor/golang.org/x/net/http2/h2demo/tmpl.go create mode 100644 vendor/golang.org/x/net/http2/h2i/README.md create mode 100644 vendor/golang.org/x/net/http2/h2i/h2i.go create mode 100644 vendor/golang.org/x/net/http2/headermap.go create mode 100644 vendor/golang.org/x/net/http2/hpack/encode.go create mode 100644 vendor/golang.org/x/net/http2/hpack/encode_test.go create mode 100644 vendor/golang.org/x/net/http2/hpack/hpack.go create mode 100644 vendor/golang.org/x/net/http2/hpack/hpack_test.go create mode 100644 vendor/golang.org/x/net/http2/hpack/huffman.go create mode 100644 vendor/golang.org/x/net/http2/hpack/tables.go create mode 100644 vendor/golang.org/x/net/http2/hpack/tables_test.go create mode 100644 vendor/golang.org/x/net/http2/http2.go create mode 100644 vendor/golang.org/x/net/http2/http2_test.go create mode 100644 vendor/golang.org/x/net/http2/not_go16.go create mode 100644 vendor/golang.org/x/net/http2/not_go17.go create mode 100644 vendor/golang.org/x/net/http2/not_go18.go create mode 100644 vendor/golang.org/x/net/http2/not_go19.go create mode 100644 vendor/golang.org/x/net/http2/pipe.go create mode 100644 vendor/golang.org/x/net/http2/pipe_test.go create mode 100644 vendor/golang.org/x/net/http2/server.go create mode 100644 vendor/golang.org/x/net/http2/server_push_test.go create mode 100644 vendor/golang.org/x/net/http2/server_test.go create mode 100644 vendor/golang.org/x/net/http2/testdata/draft-ietf-httpbis-http2.xml create mode 100644 vendor/golang.org/x/net/http2/transport.go create mode 100644 vendor/golang.org/x/net/http2/transport_test.go create mode 100644 vendor/golang.org/x/net/http2/write.go create mode 100644 vendor/golang.org/x/net/http2/writesched.go create mode 100644 vendor/golang.org/x/net/http2/writesched_priority.go create mode 100644 vendor/golang.org/x/net/http2/writesched_priority_test.go create mode 100644 vendor/golang.org/x/net/http2/writesched_random.go create mode 100644 vendor/golang.org/x/net/http2/writesched_random_test.go create mode 100644 vendor/golang.org/x/net/http2/writesched_test.go create mode 100644 vendor/golang.org/x/net/http2/z_spec_test.go create mode 100644 vendor/golang.org/x/net/icmp/dstunreach.go create mode 100644 vendor/golang.org/x/net/icmp/echo.go create mode 100644 vendor/golang.org/x/net/icmp/endpoint.go create mode 100644 vendor/golang.org/x/net/icmp/example_test.go create mode 100644 vendor/golang.org/x/net/icmp/extension.go create mode 100644 vendor/golang.org/x/net/icmp/extension_test.go create mode 100644 vendor/golang.org/x/net/icmp/helper_posix.go create mode 100644 vendor/golang.org/x/net/icmp/interface.go create mode 100644 vendor/golang.org/x/net/icmp/ipv4.go create mode 100644 vendor/golang.org/x/net/icmp/ipv4_test.go create mode 100644 vendor/golang.org/x/net/icmp/ipv6.go create mode 100644 vendor/golang.org/x/net/icmp/listen_posix.go create mode 100644 vendor/golang.org/x/net/icmp/listen_stub.go create mode 100644 vendor/golang.org/x/net/icmp/message.go create mode 100644 vendor/golang.org/x/net/icmp/message_test.go create mode 100644 vendor/golang.org/x/net/icmp/messagebody.go create mode 100644 vendor/golang.org/x/net/icmp/mpls.go create mode 100644 vendor/golang.org/x/net/icmp/multipart.go create mode 100644 vendor/golang.org/x/net/icmp/multipart_test.go create mode 100644 vendor/golang.org/x/net/icmp/packettoobig.go create mode 100644 vendor/golang.org/x/net/icmp/paramprob.go create mode 100644 vendor/golang.org/x/net/icmp/ping_test.go create mode 100644 vendor/golang.org/x/net/icmp/sys_freebsd.go create mode 100644 vendor/golang.org/x/net/icmp/timeexceeded.go create mode 100644 vendor/golang.org/x/net/idna/example_test.go create mode 100644 vendor/golang.org/x/net/idna/idna.go create mode 100644 vendor/golang.org/x/net/idna/idna_test.go create mode 100644 vendor/golang.org/x/net/idna/punycode.go create mode 100644 vendor/golang.org/x/net/idna/punycode_test.go create mode 100644 vendor/golang.org/x/net/idna/tables.go create mode 100644 vendor/golang.org/x/net/idna/trie.go create mode 100644 vendor/golang.org/x/net/idna/trieval.go create mode 100644 vendor/golang.org/x/net/internal/iana/const.go create mode 100644 vendor/golang.org/x/net/internal/iana/gen.go create mode 100644 vendor/golang.org/x/net/internal/nettest/helper_bsd.go create mode 100644 vendor/golang.org/x/net/internal/nettest/helper_nobsd.go create mode 100644 vendor/golang.org/x/net/internal/nettest/helper_posix.go create mode 100644 vendor/golang.org/x/net/internal/nettest/helper_stub.go create mode 100644 vendor/golang.org/x/net/internal/nettest/helper_unix.go create mode 100644 vendor/golang.org/x/net/internal/nettest/helper_windows.go create mode 100644 vendor/golang.org/x/net/internal/nettest/interface.go create mode 100644 vendor/golang.org/x/net/internal/nettest/rlimit.go create mode 100644 vendor/golang.org/x/net/internal/nettest/stack.go create mode 100644 vendor/golang.org/x/net/internal/socket/cmsghdr.go create mode 100644 vendor/golang.org/x/net/internal/socket/cmsghdr_bsd.go create mode 100644 vendor/golang.org/x/net/internal/socket/cmsghdr_linux_32bit.go create mode 100644 vendor/golang.org/x/net/internal/socket/cmsghdr_linux_64bit.go create mode 100644 vendor/golang.org/x/net/internal/socket/cmsghdr_solaris_64bit.go create mode 100644 vendor/golang.org/x/net/internal/socket/cmsghdr_stub.go create mode 100644 vendor/golang.org/x/net/internal/socket/defs_darwin.go create mode 100644 vendor/golang.org/x/net/internal/socket/defs_dragonfly.go create mode 100644 vendor/golang.org/x/net/internal/socket/defs_freebsd.go create mode 100644 vendor/golang.org/x/net/internal/socket/defs_linux.go create mode 100644 vendor/golang.org/x/net/internal/socket/defs_netbsd.go create mode 100644 vendor/golang.org/x/net/internal/socket/defs_openbsd.go create mode 100644 vendor/golang.org/x/net/internal/socket/defs_solaris.go create mode 100644 vendor/golang.org/x/net/internal/socket/error_unix.go create mode 100644 vendor/golang.org/x/net/internal/socket/error_windows.go create mode 100644 vendor/golang.org/x/net/internal/socket/iovec_32bit.go create mode 100644 vendor/golang.org/x/net/internal/socket/iovec_64bit.go create mode 100644 vendor/golang.org/x/net/internal/socket/iovec_solaris_64bit.go create mode 100644 vendor/golang.org/x/net/internal/socket/iovec_stub.go create mode 100644 vendor/golang.org/x/net/internal/socket/mmsghdr_stub.go create mode 100644 vendor/golang.org/x/net/internal/socket/mmsghdr_unix.go create mode 100644 vendor/golang.org/x/net/internal/socket/msghdr_bsd.go create mode 100644 vendor/golang.org/x/net/internal/socket/msghdr_bsdvar.go create mode 100644 vendor/golang.org/x/net/internal/socket/msghdr_linux.go create mode 100644 vendor/golang.org/x/net/internal/socket/msghdr_linux_32bit.go create mode 100644 vendor/golang.org/x/net/internal/socket/msghdr_linux_64bit.go create mode 100644 vendor/golang.org/x/net/internal/socket/msghdr_openbsd.go create mode 100644 vendor/golang.org/x/net/internal/socket/msghdr_solaris_64bit.go create mode 100644 vendor/golang.org/x/net/internal/socket/msghdr_stub.go create mode 100644 vendor/golang.org/x/net/internal/socket/rawconn.go create mode 100644 vendor/golang.org/x/net/internal/socket/rawconn_mmsg.go create mode 100644 vendor/golang.org/x/net/internal/socket/rawconn_msg.go create mode 100644 vendor/golang.org/x/net/internal/socket/rawconn_nommsg.go create mode 100644 vendor/golang.org/x/net/internal/socket/rawconn_nomsg.go create mode 100644 vendor/golang.org/x/net/internal/socket/rawconn_stub.go create mode 100644 vendor/golang.org/x/net/internal/socket/reflect.go create mode 100644 vendor/golang.org/x/net/internal/socket/socket.go create mode 100644 vendor/golang.org/x/net/internal/socket/socket_go1_9_test.go create mode 100644 vendor/golang.org/x/net/internal/socket/socket_test.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_bsd.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_bsdvar.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_darwin.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_dragonfly.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_linux.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_linux_386.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_linux_386.s create mode 100644 vendor/golang.org/x/net/internal/socket/sys_linux_amd64.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_linux_arm.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_linux_arm64.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_linux_mips.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_linux_mips64.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_linux_mips64le.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_linux_mipsle.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_linux_ppc64.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_linux_ppc64le.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_linux_s390x.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_linux_s390x.s create mode 100644 vendor/golang.org/x/net/internal/socket/sys_netbsd.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_posix.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_solaris.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_solaris_amd64.s create mode 100644 vendor/golang.org/x/net/internal/socket/sys_stub.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_unix.go create mode 100644 vendor/golang.org/x/net/internal/socket/sys_windows.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_darwin_386.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_darwin_amd64.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_darwin_arm.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_dragonfly_amd64.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_freebsd_386.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_freebsd_amd64.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_freebsd_arm.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_linux_386.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_linux_amd64.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_linux_arm.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_linux_arm64.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_linux_mips.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_linux_mips64.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_linux_mips64le.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_linux_mipsle.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_linux_ppc64.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_linux_ppc64le.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_linux_s390x.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_netbsd_386.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_netbsd_amd64.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_netbsd_arm.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_openbsd_386.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_openbsd_amd64.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_openbsd_arm.go create mode 100644 vendor/golang.org/x/net/internal/socket/zsys_solaris_amd64.go create mode 100644 vendor/golang.org/x/net/internal/timeseries/timeseries.go create mode 100644 vendor/golang.org/x/net/internal/timeseries/timeseries_test.go create mode 100644 vendor/golang.org/x/net/ipv4/batch.go create mode 100644 vendor/golang.org/x/net/ipv4/bpf_test.go create mode 100644 vendor/golang.org/x/net/ipv4/control.go create mode 100644 vendor/golang.org/x/net/ipv4/control_bsd.go create mode 100644 vendor/golang.org/x/net/ipv4/control_pktinfo.go create mode 100644 vendor/golang.org/x/net/ipv4/control_stub.go create mode 100644 vendor/golang.org/x/net/ipv4/control_test.go create mode 100644 vendor/golang.org/x/net/ipv4/control_unix.go create mode 100644 vendor/golang.org/x/net/ipv4/control_windows.go create mode 100644 vendor/golang.org/x/net/ipv4/defs_darwin.go create mode 100644 vendor/golang.org/x/net/ipv4/defs_dragonfly.go create mode 100644 vendor/golang.org/x/net/ipv4/defs_freebsd.go create mode 100644 vendor/golang.org/x/net/ipv4/defs_linux.go create mode 100644 vendor/golang.org/x/net/ipv4/defs_netbsd.go create mode 100644 vendor/golang.org/x/net/ipv4/defs_openbsd.go create mode 100644 vendor/golang.org/x/net/ipv4/defs_solaris.go create mode 100644 vendor/golang.org/x/net/ipv4/dgramopt.go create mode 100644 vendor/golang.org/x/net/ipv4/doc.go create mode 100644 vendor/golang.org/x/net/ipv4/endpoint.go create mode 100644 vendor/golang.org/x/net/ipv4/example_test.go create mode 100644 vendor/golang.org/x/net/ipv4/gen.go create mode 100644 vendor/golang.org/x/net/ipv4/genericopt.go create mode 100644 vendor/golang.org/x/net/ipv4/header.go create mode 100644 vendor/golang.org/x/net/ipv4/header_test.go create mode 100644 vendor/golang.org/x/net/ipv4/helper.go create mode 100644 vendor/golang.org/x/net/ipv4/iana.go create mode 100644 vendor/golang.org/x/net/ipv4/icmp.go create mode 100644 vendor/golang.org/x/net/ipv4/icmp_linux.go create mode 100644 vendor/golang.org/x/net/ipv4/icmp_stub.go create mode 100644 vendor/golang.org/x/net/ipv4/icmp_test.go create mode 100644 vendor/golang.org/x/net/ipv4/multicast_test.go create mode 100644 vendor/golang.org/x/net/ipv4/multicastlistener_test.go create mode 100644 vendor/golang.org/x/net/ipv4/multicastsockopt_test.go create mode 100644 vendor/golang.org/x/net/ipv4/packet.go create mode 100644 vendor/golang.org/x/net/ipv4/packet_go1_8.go create mode 100644 vendor/golang.org/x/net/ipv4/packet_go1_9.go create mode 100644 vendor/golang.org/x/net/ipv4/payload.go create mode 100644 vendor/golang.org/x/net/ipv4/payload_cmsg.go create mode 100644 vendor/golang.org/x/net/ipv4/payload_cmsg_go1_8.go create mode 100644 vendor/golang.org/x/net/ipv4/payload_cmsg_go1_9.go create mode 100644 vendor/golang.org/x/net/ipv4/payload_nocmsg.go create mode 100644 vendor/golang.org/x/net/ipv4/readwrite_go1_8_test.go create mode 100644 vendor/golang.org/x/net/ipv4/readwrite_go1_9_test.go create mode 100644 vendor/golang.org/x/net/ipv4/readwrite_test.go create mode 100644 vendor/golang.org/x/net/ipv4/sockopt.go create mode 100644 vendor/golang.org/x/net/ipv4/sockopt_posix.go create mode 100644 vendor/golang.org/x/net/ipv4/sockopt_stub.go create mode 100644 vendor/golang.org/x/net/ipv4/sys_asmreq.go create mode 100644 vendor/golang.org/x/net/ipv4/sys_asmreq_stub.go create mode 100644 vendor/golang.org/x/net/ipv4/sys_asmreqn.go create mode 100644 vendor/golang.org/x/net/ipv4/sys_asmreqn_stub.go create mode 100644 vendor/golang.org/x/net/ipv4/sys_bpf.go create mode 100644 vendor/golang.org/x/net/ipv4/sys_bpf_stub.go create mode 100644 vendor/golang.org/x/net/ipv4/sys_bsd.go create mode 100644 vendor/golang.org/x/net/ipv4/sys_darwin.go create mode 100644 vendor/golang.org/x/net/ipv4/sys_dragonfly.go create mode 100644 vendor/golang.org/x/net/ipv4/sys_freebsd.go create mode 100644 vendor/golang.org/x/net/ipv4/sys_linux.go create mode 100644 vendor/golang.org/x/net/ipv4/sys_solaris.go create mode 100644 vendor/golang.org/x/net/ipv4/sys_ssmreq.go create mode 100644 vendor/golang.org/x/net/ipv4/sys_ssmreq_stub.go create mode 100644 vendor/golang.org/x/net/ipv4/sys_stub.go create mode 100644 vendor/golang.org/x/net/ipv4/sys_windows.go create mode 100644 vendor/golang.org/x/net/ipv4/unicast_test.go create mode 100644 vendor/golang.org/x/net/ipv4/unicastsockopt_test.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_darwin.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_dragonfly.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_freebsd_386.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_freebsd_amd64.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_freebsd_arm.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_linux_386.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_linux_amd64.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_linux_arm.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_linux_arm64.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_linux_mips.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_linux_mips64.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_linux_mips64le.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_linux_mipsle.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_linux_ppc.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_linux_ppc64.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_linux_ppc64le.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_linux_s390x.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_netbsd.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_openbsd.go create mode 100644 vendor/golang.org/x/net/ipv4/zsys_solaris.go create mode 100644 vendor/golang.org/x/net/ipv6/batch.go create mode 100644 vendor/golang.org/x/net/ipv6/bpf_test.go create mode 100644 vendor/golang.org/x/net/ipv6/control.go create mode 100644 vendor/golang.org/x/net/ipv6/control_rfc2292_unix.go create mode 100644 vendor/golang.org/x/net/ipv6/control_rfc3542_unix.go create mode 100644 vendor/golang.org/x/net/ipv6/control_stub.go create mode 100644 vendor/golang.org/x/net/ipv6/control_test.go create mode 100644 vendor/golang.org/x/net/ipv6/control_unix.go create mode 100644 vendor/golang.org/x/net/ipv6/control_windows.go create mode 100644 vendor/golang.org/x/net/ipv6/defs_darwin.go create mode 100644 vendor/golang.org/x/net/ipv6/defs_dragonfly.go create mode 100644 vendor/golang.org/x/net/ipv6/defs_freebsd.go create mode 100644 vendor/golang.org/x/net/ipv6/defs_linux.go create mode 100644 vendor/golang.org/x/net/ipv6/defs_netbsd.go create mode 100644 vendor/golang.org/x/net/ipv6/defs_openbsd.go create mode 100644 vendor/golang.org/x/net/ipv6/defs_solaris.go create mode 100644 vendor/golang.org/x/net/ipv6/dgramopt.go create mode 100644 vendor/golang.org/x/net/ipv6/doc.go create mode 100644 vendor/golang.org/x/net/ipv6/endpoint.go create mode 100644 vendor/golang.org/x/net/ipv6/example_test.go create mode 100644 vendor/golang.org/x/net/ipv6/gen.go create mode 100644 vendor/golang.org/x/net/ipv6/genericopt.go create mode 100644 vendor/golang.org/x/net/ipv6/header.go create mode 100644 vendor/golang.org/x/net/ipv6/header_test.go create mode 100644 vendor/golang.org/x/net/ipv6/helper.go create mode 100644 vendor/golang.org/x/net/ipv6/iana.go create mode 100644 vendor/golang.org/x/net/ipv6/icmp.go create mode 100644 vendor/golang.org/x/net/ipv6/icmp_bsd.go create mode 100644 vendor/golang.org/x/net/ipv6/icmp_linux.go create mode 100644 vendor/golang.org/x/net/ipv6/icmp_solaris.go create mode 100644 vendor/golang.org/x/net/ipv6/icmp_stub.go create mode 100644 vendor/golang.org/x/net/ipv6/icmp_test.go create mode 100644 vendor/golang.org/x/net/ipv6/icmp_windows.go create mode 100644 vendor/golang.org/x/net/ipv6/mocktransponder_test.go create mode 100644 vendor/golang.org/x/net/ipv6/multicast_test.go create mode 100644 vendor/golang.org/x/net/ipv6/multicastlistener_test.go create mode 100644 vendor/golang.org/x/net/ipv6/multicastsockopt_test.go create mode 100644 vendor/golang.org/x/net/ipv6/payload.go create mode 100644 vendor/golang.org/x/net/ipv6/payload_cmsg.go create mode 100644 vendor/golang.org/x/net/ipv6/payload_cmsg_go1_8.go create mode 100644 vendor/golang.org/x/net/ipv6/payload_cmsg_go1_9.go create mode 100644 vendor/golang.org/x/net/ipv6/payload_nocmsg.go create mode 100644 vendor/golang.org/x/net/ipv6/readwrite_go1_8_test.go create mode 100644 vendor/golang.org/x/net/ipv6/readwrite_go1_9_test.go create mode 100644 vendor/golang.org/x/net/ipv6/readwrite_test.go create mode 100644 vendor/golang.org/x/net/ipv6/sockopt.go create mode 100644 vendor/golang.org/x/net/ipv6/sockopt_posix.go create mode 100644 vendor/golang.org/x/net/ipv6/sockopt_stub.go create mode 100644 vendor/golang.org/x/net/ipv6/sockopt_test.go create mode 100644 vendor/golang.org/x/net/ipv6/sys_asmreq.go create mode 100644 vendor/golang.org/x/net/ipv6/sys_asmreq_stub.go create mode 100644 vendor/golang.org/x/net/ipv6/sys_bpf.go create mode 100644 vendor/golang.org/x/net/ipv6/sys_bpf_stub.go create mode 100644 vendor/golang.org/x/net/ipv6/sys_bsd.go create mode 100644 vendor/golang.org/x/net/ipv6/sys_darwin.go create mode 100644 vendor/golang.org/x/net/ipv6/sys_freebsd.go create mode 100644 vendor/golang.org/x/net/ipv6/sys_linux.go create mode 100644 vendor/golang.org/x/net/ipv6/sys_solaris.go create mode 100644 vendor/golang.org/x/net/ipv6/sys_ssmreq.go create mode 100644 vendor/golang.org/x/net/ipv6/sys_ssmreq_stub.go create mode 100644 vendor/golang.org/x/net/ipv6/sys_stub.go create mode 100644 vendor/golang.org/x/net/ipv6/sys_windows.go create mode 100644 vendor/golang.org/x/net/ipv6/unicast_test.go create mode 100644 vendor/golang.org/x/net/ipv6/unicastsockopt_test.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_darwin.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_dragonfly.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_freebsd_386.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_freebsd_amd64.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_freebsd_arm.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_linux_386.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_linux_amd64.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_linux_arm.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_linux_arm64.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_linux_mips.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_linux_mips64.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_linux_mips64le.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_linux_mipsle.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_linux_ppc.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_linux_ppc64.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_linux_ppc64le.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_linux_s390x.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_netbsd.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_openbsd.go create mode 100644 vendor/golang.org/x/net/ipv6/zsys_solaris.go create mode 100644 vendor/golang.org/x/net/lex/httplex/httplex.go create mode 100644 vendor/golang.org/x/net/lex/httplex/httplex_test.go create mode 100644 vendor/golang.org/x/net/lif/address.go create mode 100644 vendor/golang.org/x/net/lif/address_test.go create mode 100644 vendor/golang.org/x/net/lif/binary.go create mode 100644 vendor/golang.org/x/net/lif/defs_solaris.go create mode 100644 vendor/golang.org/x/net/lif/lif.go create mode 100644 vendor/golang.org/x/net/lif/link.go create mode 100644 vendor/golang.org/x/net/lif/link_test.go create mode 100644 vendor/golang.org/x/net/lif/sys.go create mode 100644 vendor/golang.org/x/net/lif/sys_solaris_amd64.s create mode 100644 vendor/golang.org/x/net/lif/syscall.go create mode 100644 vendor/golang.org/x/net/lif/zsys_solaris_amd64.go create mode 100644 vendor/golang.org/x/net/nettest/conntest.go create mode 100644 vendor/golang.org/x/net/nettest/conntest_go16.go create mode 100644 vendor/golang.org/x/net/nettest/conntest_go17.go create mode 100644 vendor/golang.org/x/net/nettest/conntest_test.go create mode 100644 vendor/golang.org/x/net/netutil/listen.go create mode 100644 vendor/golang.org/x/net/netutil/listen_test.go create mode 100644 vendor/golang.org/x/net/proxy/direct.go create mode 100644 vendor/golang.org/x/net/proxy/per_host.go create mode 100644 vendor/golang.org/x/net/proxy/per_host_test.go create mode 100644 vendor/golang.org/x/net/proxy/proxy.go create mode 100644 vendor/golang.org/x/net/proxy/proxy_test.go create mode 100644 vendor/golang.org/x/net/proxy/socks5.go create mode 100644 vendor/golang.org/x/net/publicsuffix/gen.go create mode 100644 vendor/golang.org/x/net/publicsuffix/list.go create mode 100644 vendor/golang.org/x/net/publicsuffix/list_test.go create mode 100644 vendor/golang.org/x/net/publicsuffix/table.go create mode 100644 vendor/golang.org/x/net/publicsuffix/table_test.go create mode 100644 vendor/golang.org/x/net/route/address.go create mode 100644 vendor/golang.org/x/net/route/address_darwin_test.go create mode 100644 vendor/golang.org/x/net/route/address_test.go create mode 100644 vendor/golang.org/x/net/route/binary.go create mode 100644 vendor/golang.org/x/net/route/defs_darwin.go create mode 100644 vendor/golang.org/x/net/route/defs_dragonfly.go create mode 100644 vendor/golang.org/x/net/route/defs_freebsd.go create mode 100644 vendor/golang.org/x/net/route/defs_netbsd.go create mode 100644 vendor/golang.org/x/net/route/defs_openbsd.go create mode 100644 vendor/golang.org/x/net/route/interface.go create mode 100644 vendor/golang.org/x/net/route/interface_announce.go create mode 100644 vendor/golang.org/x/net/route/interface_classic.go create mode 100644 vendor/golang.org/x/net/route/interface_freebsd.go create mode 100644 vendor/golang.org/x/net/route/interface_multicast.go create mode 100644 vendor/golang.org/x/net/route/interface_openbsd.go create mode 100644 vendor/golang.org/x/net/route/message.go create mode 100644 vendor/golang.org/x/net/route/message_darwin_test.go create mode 100644 vendor/golang.org/x/net/route/message_freebsd_test.go create mode 100644 vendor/golang.org/x/net/route/message_test.go create mode 100644 vendor/golang.org/x/net/route/route.go create mode 100644 vendor/golang.org/x/net/route/route_classic.go create mode 100644 vendor/golang.org/x/net/route/route_openbsd.go create mode 100644 vendor/golang.org/x/net/route/route_test.go create mode 100644 vendor/golang.org/x/net/route/sys.go create mode 100644 vendor/golang.org/x/net/route/sys_darwin.go create mode 100644 vendor/golang.org/x/net/route/sys_dragonfly.go create mode 100644 vendor/golang.org/x/net/route/sys_freebsd.go create mode 100644 vendor/golang.org/x/net/route/sys_netbsd.go create mode 100644 vendor/golang.org/x/net/route/sys_openbsd.go create mode 100644 vendor/golang.org/x/net/route/syscall.go create mode 100644 vendor/golang.org/x/net/route/zsys_darwin.go create mode 100644 vendor/golang.org/x/net/route/zsys_dragonfly.go create mode 100644 vendor/golang.org/x/net/route/zsys_freebsd_386.go create mode 100644 vendor/golang.org/x/net/route/zsys_freebsd_amd64.go create mode 100644 vendor/golang.org/x/net/route/zsys_freebsd_arm.go create mode 100644 vendor/golang.org/x/net/route/zsys_netbsd.go create mode 100644 vendor/golang.org/x/net/route/zsys_openbsd.go create mode 100644 vendor/golang.org/x/net/trace/events.go create mode 100644 vendor/golang.org/x/net/trace/histogram.go create mode 100644 vendor/golang.org/x/net/trace/histogram_test.go create mode 100644 vendor/golang.org/x/net/trace/trace.go create mode 100644 vendor/golang.org/x/net/trace/trace_go16.go create mode 100644 vendor/golang.org/x/net/trace/trace_go17.go create mode 100644 vendor/golang.org/x/net/trace/trace_test.go create mode 100644 vendor/golang.org/x/net/webdav/file.go create mode 100644 vendor/golang.org/x/net/webdav/file_go1.6.go create mode 100644 vendor/golang.org/x/net/webdav/file_go1.7.go create mode 100644 vendor/golang.org/x/net/webdav/file_test.go create mode 100644 vendor/golang.org/x/net/webdav/if.go create mode 100644 vendor/golang.org/x/net/webdav/if_test.go create mode 100644 vendor/golang.org/x/net/webdav/internal/xml/README create mode 100644 vendor/golang.org/x/net/webdav/internal/xml/atom_test.go create mode 100644 vendor/golang.org/x/net/webdav/internal/xml/example_test.go create mode 100644 vendor/golang.org/x/net/webdav/internal/xml/marshal.go create mode 100644 vendor/golang.org/x/net/webdav/internal/xml/marshal_test.go create mode 100644 vendor/golang.org/x/net/webdav/internal/xml/read.go create mode 100644 vendor/golang.org/x/net/webdav/internal/xml/read_test.go create mode 100644 vendor/golang.org/x/net/webdav/internal/xml/typeinfo.go create mode 100644 vendor/golang.org/x/net/webdav/internal/xml/xml.go create mode 100644 vendor/golang.org/x/net/webdav/internal/xml/xml_test.go create mode 100644 vendor/golang.org/x/net/webdav/litmus_test_server.go create mode 100644 vendor/golang.org/x/net/webdav/lock.go create mode 100644 vendor/golang.org/x/net/webdav/lock_test.go create mode 100644 vendor/golang.org/x/net/webdav/prop.go create mode 100644 vendor/golang.org/x/net/webdav/prop_test.go create mode 100644 vendor/golang.org/x/net/webdav/webdav.go create mode 100644 vendor/golang.org/x/net/webdav/webdav_test.go create mode 100644 vendor/golang.org/x/net/webdav/xml.go create mode 100644 vendor/golang.org/x/net/webdav/xml_test.go create mode 100644 vendor/golang.org/x/net/websocket/client.go create mode 100644 vendor/golang.org/x/net/websocket/dial.go create mode 100644 vendor/golang.org/x/net/websocket/dial_test.go create mode 100644 vendor/golang.org/x/net/websocket/exampledial_test.go create mode 100644 vendor/golang.org/x/net/websocket/examplehandler_test.go create mode 100644 vendor/golang.org/x/net/websocket/hybi.go create mode 100644 vendor/golang.org/x/net/websocket/hybi_test.go create mode 100644 vendor/golang.org/x/net/websocket/server.go create mode 100644 vendor/golang.org/x/net/websocket/websocket.go create mode 100644 vendor/golang.org/x/net/websocket/websocket_test.go create mode 100644 vendor/golang.org/x/net/xsrftoken/xsrf.go create mode 100644 vendor/golang.org/x/net/xsrftoken/xsrf_test.go create mode 100644 vendor/golang.org/x/sys/.gitattributes create mode 100644 vendor/golang.org/x/sys/.gitignore create mode 100644 vendor/golang.org/x/sys/AUTHORS create mode 100644 vendor/golang.org/x/sys/CONTRIBUTING.md create mode 100644 vendor/golang.org/x/sys/CONTRIBUTORS create mode 100644 vendor/golang.org/x/sys/LICENSE create mode 100644 vendor/golang.org/x/sys/PATENTS create mode 100644 vendor/golang.org/x/sys/README create mode 100644 vendor/golang.org/x/sys/codereview.cfg create mode 100644 vendor/golang.org/x/sys/plan9/asm.s create mode 100644 vendor/golang.org/x/sys/plan9/asm_plan9_386.s create mode 100644 vendor/golang.org/x/sys/plan9/asm_plan9_amd64.s create mode 100644 vendor/golang.org/x/sys/plan9/const_plan9.go create mode 100644 vendor/golang.org/x/sys/plan9/dir_plan9.go create mode 100644 vendor/golang.org/x/sys/plan9/env_plan9.go create mode 100644 vendor/golang.org/x/sys/plan9/env_unset.go create mode 100644 vendor/golang.org/x/sys/plan9/errors_plan9.go create mode 100755 vendor/golang.org/x/sys/plan9/mksyscall.pl create mode 100644 vendor/golang.org/x/sys/plan9/pwd_go15_plan9.go create mode 100644 vendor/golang.org/x/sys/plan9/pwd_plan9.go create mode 100644 vendor/golang.org/x/sys/plan9/race.go create mode 100644 vendor/golang.org/x/sys/plan9/race0.go create mode 100644 vendor/golang.org/x/sys/plan9/str.go create mode 100644 vendor/golang.org/x/sys/plan9/syscall.go create mode 100644 vendor/golang.org/x/sys/plan9/syscall_plan9.go create mode 100644 vendor/golang.org/x/sys/plan9/syscall_test.go create mode 100644 vendor/golang.org/x/sys/plan9/zsyscall_plan9_386.go create mode 100644 vendor/golang.org/x/sys/plan9/zsyscall_plan9_amd64.go create mode 100644 vendor/golang.org/x/sys/plan9/zsysnum_plan9.go create mode 100644 vendor/golang.org/x/sys/unix/.gitignore create mode 100644 vendor/golang.org/x/sys/unix/README.md create mode 100644 vendor/golang.org/x/sys/unix/asm_darwin_386.s create mode 100644 vendor/golang.org/x/sys/unix/asm_darwin_amd64.s create mode 100644 vendor/golang.org/x/sys/unix/asm_darwin_arm.s create mode 100644 vendor/golang.org/x/sys/unix/asm_darwin_arm64.s create mode 100644 vendor/golang.org/x/sys/unix/asm_dragonfly_amd64.s create mode 100644 vendor/golang.org/x/sys/unix/asm_freebsd_386.s create mode 100644 vendor/golang.org/x/sys/unix/asm_freebsd_amd64.s create mode 100644 vendor/golang.org/x/sys/unix/asm_freebsd_arm.s create mode 100644 vendor/golang.org/x/sys/unix/asm_linux_386.s create mode 100644 vendor/golang.org/x/sys/unix/asm_linux_amd64.s create mode 100644 vendor/golang.org/x/sys/unix/asm_linux_arm.s create mode 100644 vendor/golang.org/x/sys/unix/asm_linux_arm64.s create mode 100644 vendor/golang.org/x/sys/unix/asm_linux_mips64x.s create mode 100644 vendor/golang.org/x/sys/unix/asm_linux_mipsx.s create mode 100644 vendor/golang.org/x/sys/unix/asm_linux_ppc64x.s create mode 100644 vendor/golang.org/x/sys/unix/asm_linux_s390x.s create mode 100644 vendor/golang.org/x/sys/unix/asm_netbsd_386.s create mode 100644 vendor/golang.org/x/sys/unix/asm_netbsd_amd64.s create mode 100644 vendor/golang.org/x/sys/unix/asm_netbsd_arm.s create mode 100644 vendor/golang.org/x/sys/unix/asm_openbsd_386.s create mode 100644 vendor/golang.org/x/sys/unix/asm_openbsd_amd64.s create mode 100644 vendor/golang.org/x/sys/unix/asm_openbsd_arm.s create mode 100644 vendor/golang.org/x/sys/unix/asm_solaris_amd64.s create mode 100644 vendor/golang.org/x/sys/unix/bluetooth_linux.go create mode 100644 vendor/golang.org/x/sys/unix/cap_freebsd.go create mode 100644 vendor/golang.org/x/sys/unix/constants.go create mode 100644 vendor/golang.org/x/sys/unix/creds_test.go create mode 100644 vendor/golang.org/x/sys/unix/dev_darwin.go create mode 100644 vendor/golang.org/x/sys/unix/dev_darwin_test.go create mode 100644 vendor/golang.org/x/sys/unix/dev_dragonfly.go create mode 100644 vendor/golang.org/x/sys/unix/dev_dragonfly_test.go create mode 100644 vendor/golang.org/x/sys/unix/dev_freebsd.go create mode 100644 vendor/golang.org/x/sys/unix/dev_linux.go create mode 100644 vendor/golang.org/x/sys/unix/dev_linux_test.go create mode 100644 vendor/golang.org/x/sys/unix/dev_netbsd.go create mode 100644 vendor/golang.org/x/sys/unix/dev_netbsd_test.go create mode 100644 vendor/golang.org/x/sys/unix/dev_openbsd.go create mode 100644 vendor/golang.org/x/sys/unix/dev_openbsd_test.go create mode 100644 vendor/golang.org/x/sys/unix/dirent.go create mode 100644 vendor/golang.org/x/sys/unix/endian_big.go create mode 100644 vendor/golang.org/x/sys/unix/endian_little.go create mode 100644 vendor/golang.org/x/sys/unix/env_unix.go create mode 100644 vendor/golang.org/x/sys/unix/env_unset.go create mode 100644 vendor/golang.org/x/sys/unix/errors_freebsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/errors_freebsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/errors_freebsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/export_test.go create mode 100644 vendor/golang.org/x/sys/unix/file_unix.go create mode 100644 vendor/golang.org/x/sys/unix/flock.go create mode 100644 vendor/golang.org/x/sys/unix/flock_linux_32bit.go create mode 100644 vendor/golang.org/x/sys/unix/gccgo.go create mode 100644 vendor/golang.org/x/sys/unix/gccgo_c.c create mode 100644 vendor/golang.org/x/sys/unix/gccgo_linux_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/linux/Dockerfile create mode 100644 vendor/golang.org/x/sys/unix/linux/mkall.go create mode 100755 vendor/golang.org/x/sys/unix/linux/mksysnum.pl create mode 100644 vendor/golang.org/x/sys/unix/linux/types.go create mode 100644 vendor/golang.org/x/sys/unix/mkpost.go create mode 100755 vendor/golang.org/x/sys/unix/mksyscall.pl create mode 100755 vendor/golang.org/x/sys/unix/mksyscall_solaris.pl create mode 100755 vendor/golang.org/x/sys/unix/mksysctl_openbsd.pl create mode 100755 vendor/golang.org/x/sys/unix/mksysnum_darwin.pl create mode 100755 vendor/golang.org/x/sys/unix/mksysnum_dragonfly.pl create mode 100755 vendor/golang.org/x/sys/unix/mksysnum_freebsd.pl create mode 100755 vendor/golang.org/x/sys/unix/mksysnum_netbsd.pl create mode 100755 vendor/golang.org/x/sys/unix/mksysnum_openbsd.pl create mode 100644 vendor/golang.org/x/sys/unix/mmap_unix_test.go create mode 100644 vendor/golang.org/x/sys/unix/openbsd_pledge.go create mode 100644 vendor/golang.org/x/sys/unix/openbsd_test.go create mode 100644 vendor/golang.org/x/sys/unix/pagesize_unix.go create mode 100644 vendor/golang.org/x/sys/unix/race.go create mode 100644 vendor/golang.org/x/sys/unix/race0.go create mode 100644 vendor/golang.org/x/sys/unix/sockcmsg_linux.go create mode 100644 vendor/golang.org/x/sys/unix/sockcmsg_unix.go create mode 100644 vendor/golang.org/x/sys/unix/str.go create mode 100644 vendor/golang.org/x/sys/unix/syscall.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_bsd.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_bsd_test.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_darwin.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_darwin_386.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_darwin_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_darwin_arm.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_darwin_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_dragonfly.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_dragonfly_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_freebsd.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_freebsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_freebsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_freebsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_freebsd_test.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_386.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_amd64_gc.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_arm.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_mips64x.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_mipsx.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_ppc64x.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_s390x.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_sparc64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_test.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_netbsd.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_netbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_netbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_netbsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_no_getwd.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_openbsd.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_openbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_openbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_openbsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_solaris.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_solaris_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_solaris_test.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_test.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_unix.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_unix_gc.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_unix_test.go create mode 100644 vendor/golang.org/x/sys/unix/types_darwin.go create mode 100644 vendor/golang.org/x/sys/unix/types_dragonfly.go create mode 100644 vendor/golang.org/x/sys/unix/types_freebsd.go create mode 100644 vendor/golang.org/x/sys/unix/types_netbsd.go create mode 100644 vendor/golang.org/x/sys/unix/types_openbsd.go create mode 100644 vendor/golang.org/x/sys/unix/types_solaris.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_darwin_386.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_darwin_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_dragonfly_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_freebsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_freebsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_freebsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_386.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_mips.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_netbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_netbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_netbsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_openbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_openbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_openbsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_solaris_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_darwin_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_darwin_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_dragonfly_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_freebsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_freebsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_mips.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_mips64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_mips64le.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_mipsle.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64le.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_s390x.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_sparc64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_netbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_netbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_solaris_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysctl_openbsd.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_darwin_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_darwin_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_darwin_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_darwin_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_dragonfly_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_freebsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_freebsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_freebsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_netbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_netbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_netbsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_openbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_openbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_openbsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_solaris_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_darwin_386.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_darwin_arm.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_dragonfly_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_freebsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_freebsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_freebsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_386.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_arm.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_mips.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_netbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_netbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_netbsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_openbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_openbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_openbsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_solaris_amd64.go create mode 100644 vendor/golang.org/x/sys/windows/asm_windows_386.s create mode 100644 vendor/golang.org/x/sys/windows/asm_windows_amd64.s create mode 100644 vendor/golang.org/x/sys/windows/dll_windows.go create mode 100644 vendor/golang.org/x/sys/windows/env_unset.go create mode 100644 vendor/golang.org/x/sys/windows/env_windows.go create mode 100644 vendor/golang.org/x/sys/windows/eventlog.go create mode 100644 vendor/golang.org/x/sys/windows/exec_windows.go create mode 100644 vendor/golang.org/x/sys/windows/memory_windows.go create mode 100644 vendor/golang.org/x/sys/windows/mksyscall.go create mode 100644 vendor/golang.org/x/sys/windows/race.go create mode 100644 vendor/golang.org/x/sys/windows/race0.go create mode 100644 vendor/golang.org/x/sys/windows/registry/export_test.go create mode 100644 vendor/golang.org/x/sys/windows/registry/key.go create mode 100644 vendor/golang.org/x/sys/windows/registry/mksyscall.go create mode 100644 vendor/golang.org/x/sys/windows/registry/registry_test.go create mode 100644 vendor/golang.org/x/sys/windows/registry/syscall.go create mode 100644 vendor/golang.org/x/sys/windows/registry/value.go create mode 100644 vendor/golang.org/x/sys/windows/registry/zsyscall_windows.go create mode 100644 vendor/golang.org/x/sys/windows/security_windows.go create mode 100644 vendor/golang.org/x/sys/windows/service.go create mode 100644 vendor/golang.org/x/sys/windows/str.go create mode 100644 vendor/golang.org/x/sys/windows/svc/debug/log.go create mode 100644 vendor/golang.org/x/sys/windows/svc/debug/service.go create mode 100644 vendor/golang.org/x/sys/windows/svc/event.go create mode 100644 vendor/golang.org/x/sys/windows/svc/eventlog/install.go create mode 100644 vendor/golang.org/x/sys/windows/svc/eventlog/log.go create mode 100644 vendor/golang.org/x/sys/windows/svc/eventlog/log_test.go create mode 100644 vendor/golang.org/x/sys/windows/svc/example/beep.go create mode 100644 vendor/golang.org/x/sys/windows/svc/example/install.go create mode 100644 vendor/golang.org/x/sys/windows/svc/example/main.go create mode 100644 vendor/golang.org/x/sys/windows/svc/example/manage.go create mode 100644 vendor/golang.org/x/sys/windows/svc/example/service.go create mode 100644 vendor/golang.org/x/sys/windows/svc/go12.c create mode 100644 vendor/golang.org/x/sys/windows/svc/go12.go create mode 100644 vendor/golang.org/x/sys/windows/svc/go13.go create mode 100644 vendor/golang.org/x/sys/windows/svc/mgr/config.go create mode 100644 vendor/golang.org/x/sys/windows/svc/mgr/mgr.go create mode 100644 vendor/golang.org/x/sys/windows/svc/mgr/mgr_test.go create mode 100644 vendor/golang.org/x/sys/windows/svc/mgr/service.go create mode 100644 vendor/golang.org/x/sys/windows/svc/security.go create mode 100644 vendor/golang.org/x/sys/windows/svc/service.go create mode 100644 vendor/golang.org/x/sys/windows/svc/svc_test.go create mode 100644 vendor/golang.org/x/sys/windows/svc/sys_386.s create mode 100644 vendor/golang.org/x/sys/windows/svc/sys_amd64.s create mode 100644 vendor/golang.org/x/sys/windows/syscall.go create mode 100644 vendor/golang.org/x/sys/windows/syscall_test.go create mode 100644 vendor/golang.org/x/sys/windows/syscall_windows.go create mode 100644 vendor/golang.org/x/sys/windows/syscall_windows_test.go create mode 100644 vendor/golang.org/x/sys/windows/types_windows.go create mode 100644 vendor/golang.org/x/sys/windows/types_windows_386.go create mode 100644 vendor/golang.org/x/sys/windows/types_windows_amd64.go create mode 100644 vendor/golang.org/x/sys/windows/zsyscall_windows.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8fc9c85 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.idea +*.iml +*.sh +sage diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..c08e2a3 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,24 @@ +image: golang:latest + +stages: + - test + - build + +before_script: + - go get -u github.com/golang/dep/cmd/dep + - export GOPATH=$(dirname $CI_PROJECT_DIR)/go + - mkdir -p $GOPATH/src + - cd $GOPATH/src + - ln -s $CI_PROJECT_DIR + - cd $CI_PROJECT_NAME + - dep ensure -update + +test: + stage: test + script: + - go test + +build: + stage: build + script: + - go build diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..3f99ba7 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,69 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + branch = "master" + name = "github.com/BurntSushi/toml" + packages = ["."] + revision = "a368813c5e648fee92e5f6c30e3944ff9d5e8895" + +[[projects]] + name = "github.com/coreos/bbolt" + packages = ["."] + revision = "583e8937c61f1af6513608ccc75c97b6abdf4ff9" + version = "v1.3.0" + +[[projects]] + branch = "master" + name = "github.com/dustin/go-humanize" + packages = ["."] + revision = "79e699ccd02f240a1f1fbbdcee7e64c1c12e41aa" + +[[projects]] + branch = "master" + name = "github.com/fluffle/goirc" + packages = ["client","logging","state"] + revision = "c981f8f5680c85268c75c68e4945abc55ba70de0" + +[[projects]] + name = "github.com/golang/mock" + packages = ["gomock"] + revision = "13f360950a79f5864a972c786a10a50e44b69541" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "github.com/mandykoh/keva" + packages = ["."] + revision = "443c1447fc51d6502cc5cb82d810014baba00b5b" + +[[projects]] + branch = "master" + name = "github.com/mandykoh/symlock" + packages = ["."] + revision = "0362cd091b6b627bf9552d87ba15956d5e8bde32" + +[[projects]] + branch = "bbolt" + name = "github.com/tslocum/markov" + packages = ["."] + revision = "eb8ca4c563d859dd72df9334f484912ef7fb7a29" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = ["proxy"] + revision = "8351a756f30f1297fe94bbf4b767ec589c6ea6d0" + +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = ["unix"] + revision = "062cd7e4e68206d8bab9b18396626e855c992658" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "96a685db21fdef56feccf61e90a9dd578034d37eb1fa50a52dd1f8bbf34dc30b" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..b3fce76 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,38 @@ + +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" + + +[[constraint]] + branch = "master" + name = "github.com/BurntSushi/toml" + +[[constraint]] + branch = "master" + name = "github.com/dustin/go-humanize" + +[[constraint]] + branch = "master" + name = "github.com/fluffle/goirc" + +[[constraint]] + branch = "bbolt" + name = "github.com/tslocum/markov" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9cecc1d --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + {one line to give the program's name and a brief idea of what it does.} + Copyright (C) {year} {name of author} + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + {project} Copyright (C) {year} {fullname} + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md new file mode 100644 index 0000000..c01dac0 --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +# Sage + +[![pipeline status](https://gitlab.com/tslocum/sage/badges/master/pipeline.svg)](https://gitlab.com/tslocum/sage/commits/master) +[![godoc](https://godoc.org/gitlab.com/tslocum/sage?status.svg)](https://godoc.org/gitlab.com/tslocum/sage) + +Documentation coming soon... diff --git a/sage.default.conf b/sage.default.conf new file mode 100644 index 0000000..b285d23 --- /dev/null +++ b/sage.default.conf @@ -0,0 +1,5 @@ +Nick="sage" +NickPassword="hunter2" +Name="Markov Fembot" +Server="irc.freenode.net:6667" +Channels=["#jp","#saovq"] diff --git a/sage.go b/sage.go new file mode 100644 index 0000000..a891d75 --- /dev/null +++ b/sage.go @@ -0,0 +1,377 @@ +// sage - Markov chain IRC bot +// https://gitlab.com/tslocum/sage +// Written by Trevor 'tee' Slocum +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package main + +import ( + "bufio" + "flag" + "fmt" + "log" + "math/rand" + "net/http" + _ "net/http/pprof" + "os" + "os/signal" + "path" + "regexp" + "strconv" + "strings" + "syscall" + "time" + + "github.com/BurntSushi/toml" + "github.com/dustin/go-humanize" + irc "github.com/fluffle/goirc/client" + "github.com/tslocum/markov" +) + +type Config struct { + Server string + Nick string + NickPassword string + Ident string + Name string + Channels []string + + MarkovOrder int + MarkovWords int + MarkovTimeout int + + DebugPort int +} + +var config *Config +var self *markov.BoltTableStore +var memory *markov.Accumulator +var brain *markov.Model +var experience = make(chan string, 100) +var client *irc.Conn + +var validword = regexp.MustCompile("^[a-zA-Z]([a-zA-Z \\-'/]+[a-zA-Z])?$") +var invalidchars = regexp.MustCompile("[^a-zA-Z0-9 \\-'/]+") +var commonwords = []string{"a", "an", "at", "are", "arent", "and", "is", "isnt", "not", "of", "i", "you", "he", "she", "him", "her", "his", "hers", "they", "them", "theirs", "us", "our", "ours", "get", "got", "it", "in", "if", "of", "or", "on", "just", "no", "yes", "yeah", "ya", "yea", "yep", "yup"} +var dataDir = flag.String("data", "", "Data directory (contains sage.conf, memories are stored here)") +var importFile = flag.String("import", "", "Import plaintext file into memory") + +func loadConfig() { + cfile := "a new data folder\nThen supply the folder path: sage -data /home/sage/data" + if *dataDir != "" { + cfile = path.Join(*dataDir, "sage.conf") + } + nonexistmsg := fmt.Sprintf("Error! Unable to read sage.conf, please copy sage.default.conf to %s", cfile) + var err error + + config = new(Config) + if *dataDir != "" { + if _, err = os.Stat(cfile); err == nil { + if _, err = toml.DecodeFile(cfile, &config); err != nil { + log.Fatalf("Failed to read %s: %v", cfile, err) + } + + if config.Server == "" || config.Nick == "" { + log.Fatal("Server and Nick parameters in sage.conf are required") + } + } else { + log.Fatalf("%s\n%v", nonexistmsg, err) + } + } else { + log.Fatal(nonexistmsg) + } + + if config.MarkovOrder <= 0 { + config.MarkovOrder = 1 + } + if config.MarkovWords <= 0 { + config.MarkovWords = 1 + } + if config.MarkovTimeout <= 0 { + config.MarkovTimeout = 1000 + } + + if config.DebugPort > 0 { + go http.ListenAndServe(":"+strconv.Itoa(config.DebugPort), nil) + } +} + +func hear(message string) { + experience <- message +} + +func learn() { + var m []string + var valid bool + var err error + + for message := range experience { + valid = false + + m = strings.Split(stripCodes(strings.ToLower(message)), " ") + for _, tidbit := range m { + if !validword.MatchString(tidbit) { + continue // Contains number or other invalid char + } else if tidbit == config.Nick { + continue + } + + err = memory.Add(tidbit) + if err != nil { + log.Fatalf("Failed to add memory: %v", err) + } + + valid = true + } + + if valid { + err = memory.Add("") + if err != nil { + log.Fatalf("Failed to add memory: %v", err) + } + } + } +} + +func respond(message string) string { + message = strings.ToLower(message) + pieces := strings.Split(stripCommonWords(message), " ") + timeout := time.After(time.Duration(config.MarkovTimeout) * time.Millisecond) + + var thought []string + var response string + var perspective *markov.Generator +Vocalize: + for { + thought = nil + perspective = markov.NewGenerator(brain, uint(config.MarkovOrder), rand.New(rand.NewSource(time.Now().UTC().UnixNano()))) + for { + tidbit, err := perspective.Get() + if err != nil { + log.Fatalf("Unable to think: %v", err) + } + if tidbit == "" { + break + } + + thought = append(thought, tidbit) + } + + if len(thought) >= config.MarkovWords { + if len(pieces) == 0 || pieces[0] == "" { + break Vocalize + } else if response == "" { + response = strings.Join(thought, " ") + } else { + for _, piece := range pieces { + for _, tidbit := range thought { + if tidbit == piece { + break Vocalize + } + } + } + } + } + + select { + case <-timeout: + break Vocalize + default: + } + } + + if len(thought) >= config.MarkovWords { + return strings.Join(thought, " ") + } + return response +} + +func containsNick(message string) bool { + message = stripCodes(strings.ToLower(message)) + nick := strings.ToLower(config.Nick) + + m := strings.Split(message, " ") + for _, mword := range m { + if mword == nick { + return true + } + } + + return false +} + +func stripCodes(message string) string { + return invalidchars.ReplaceAllString(message, "") +} + +func stripCommonWords(message string) string { + nick := strings.ToLower(config.Nick) + m := strings.Split(message, " ") + for i, mword := range m { + if mword == nick { + m[i] = "" + continue + } + + for _, word := range commonwords { + if word == mword { + m[i] = "" + } + } + } + + return strings.Join(m, " ") +} + +func importMemories() { + if *importFile != "" { + if _, err := os.Stat(*importFile); err == nil { + lines, err := os.Open(*importFile) + defer lines.Close() + + if err == nil { + var line string + linecount := int64(0) + self.Bolt.NoSync = true + + log.Printf("Importing %s into memory...", *importFile) + scanner := bufio.NewScanner(lines) + for scanner.Scan() { + linecount++ + + line = strings.TrimSpace(scanner.Text()) + if line != "" { + hear(line) + + if linecount%1000 == 0 { + log.Printf("Import progress: " + humanize.Comma(linecount) + " lines") + } + } + } + + self.Bolt.NoSync = false + self.Bolt.Sync() + log.Printf("Imported %s lines into memory", humanize.Comma(linecount)) + } else { + log.Fatalf("Failed to read import file %s", *importFile) + } + } else { + log.Fatalf("Import file %s does not exist", *importFile) + } + } +} + +func saveMemories() { + for { + time.Sleep(1 * time.Hour) + if err := self.Bolt.Sync(); err != nil { + log.Printf("Error! Unable to write memories to file: %v", err) + } + } +} + +func terminate() { + self.Close() + os.Exit(0) +} + +func main() { + flag.Parse() + rand.Seed(time.Now().UTC().UnixNano()) + var err error + + // Intend + loadConfig() + + // Remember + dbfile := path.Join(*dataDir, "sage.db") + self, err = markov.NewBoltTableStore(dbfile) + if err != nil { + log.Fatalf("Failed to open %s: %v", dbfile, err) + } + go learn() + go saveMemories() + + // Become + brain = markov.NewModel(self) + memory = markov.NewAccumulator(brain, uint(config.MarkovOrder)) + go importMemories() + + // Explore + ident := config.Ident + if ident == "" { + ident = config.Nick + } + name := config.Name + if name == "" { + name = config.Nick + } + cfg := irc.NewConfig(config.Nick, ident, name) + cfg.Server = config.Server + cfg.Version = "sage https://gitlab.com/tslocum/sage" + cfg.NewNick = func(n string) string { return n + "^" } + client = irc.Client(cfg) + + client.HandleFunc(irc.CONNECTED, + func(conn *irc.Conn, line *irc.Line) { + log.Println("Connected!") + + if config.NickPassword != "" { + conn.Privmsg("NickServ", "IDENTIFY "+config.NickPassword) + } + + for _, channel := range config.Channels { + conn.Join(channel) + } + }) + + client.HandleFunc(irc.PRIVMSG, + func(conn *irc.Conn, line *irc.Line) { + channel := line.Args[0] + message := line.Args[1] + + hear(message) + + if containsNick(message) || rand.Intn(10) == 7 { + client.Privmsg(channel, respond(message)) + } + }) + + quit := make(chan bool) + client.HandleFunc(irc.DISCONNECTED, + func(conn *irc.Conn, line *irc.Line) { + quit <- true + }) + + termc := make(chan os.Signal, 2) + signal.Notify(termc, os.Interrupt, syscall.SIGTERM) + go func() { + <-termc + terminate() + }() + + for { + log.Printf("Connecting to %s as %s...", config.Server, config.Nick) + if err := client.Connect(); err != nil { + log.Printf("Error! Unable to connect: %v", err.Error()) + } + + <-quit + log.Println("Disconnected...") + time.Sleep(30 * time.Second) + } + + terminate() +} diff --git a/vendor/github.com/BurntSushi/toml/.gitignore b/vendor/github.com/BurntSushi/toml/.gitignore new file mode 100644 index 0000000..0cd3800 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/.gitignore @@ -0,0 +1,5 @@ +TAGS +tags +.*.swp +tomlcheck/tomlcheck +toml.test diff --git a/vendor/github.com/BurntSushi/toml/.travis.yml b/vendor/github.com/BurntSushi/toml/.travis.yml new file mode 100644 index 0000000..8b8afc4 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/.travis.yml @@ -0,0 +1,15 @@ +language: go +go: + - 1.1 + - 1.2 + - 1.3 + - 1.4 + - 1.5 + - 1.6 + - tip +install: + - go install ./... + - go get github.com/BurntSushi/toml-test +script: + - export PATH="$PATH:$HOME/gopath/bin" + - make test diff --git a/vendor/github.com/BurntSushi/toml/COMPATIBLE b/vendor/github.com/BurntSushi/toml/COMPATIBLE new file mode 100644 index 0000000..6efcfd0 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/COMPATIBLE @@ -0,0 +1,3 @@ +Compatible with TOML version +[v0.4.0](https://github.com/toml-lang/toml/blob/v0.4.0/versions/en/toml-v0.4.0.md) + diff --git a/vendor/github.com/BurntSushi/toml/COPYING b/vendor/github.com/BurntSushi/toml/COPYING new file mode 100644 index 0000000..01b5743 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/COPYING @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2013 TOML authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/BurntSushi/toml/Makefile b/vendor/github.com/BurntSushi/toml/Makefile new file mode 100644 index 0000000..3600848 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/Makefile @@ -0,0 +1,19 @@ +install: + go install ./... + +test: install + go test -v + toml-test toml-test-decoder + toml-test -encoder toml-test-encoder + +fmt: + gofmt -w *.go */*.go + colcheck *.go */*.go + +tags: + find ./ -name '*.go' -print0 | xargs -0 gotags > TAGS + +push: + git push origin master + git push github master + diff --git a/vendor/github.com/BurntSushi/toml/README.md b/vendor/github.com/BurntSushi/toml/README.md new file mode 100644 index 0000000..7c1b37e --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/README.md @@ -0,0 +1,218 @@ +## TOML parser and encoder for Go with reflection + +TOML stands for Tom's Obvious, Minimal Language. This Go package provides a +reflection interface similar to Go's standard library `json` and `xml` +packages. This package also supports the `encoding.TextUnmarshaler` and +`encoding.TextMarshaler` interfaces so that you can define custom data +representations. (There is an example of this below.) + +Spec: https://github.com/toml-lang/toml + +Compatible with TOML version +[v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md) + +Documentation: https://godoc.org/github.com/BurntSushi/toml + +Installation: + +```bash +go get github.com/BurntSushi/toml +``` + +Try the toml validator: + +```bash +go get github.com/BurntSushi/toml/cmd/tomlv +tomlv some-toml-file.toml +``` + +[![Build Status](https://travis-ci.org/BurntSushi/toml.svg?branch=master)](https://travis-ci.org/BurntSushi/toml) [![GoDoc](https://godoc.org/github.com/BurntSushi/toml?status.svg)](https://godoc.org/github.com/BurntSushi/toml) + +### Testing + +This package passes all tests in +[toml-test](https://github.com/BurntSushi/toml-test) for both the decoder +and the encoder. + +### Examples + +This package works similarly to how the Go standard library handles `XML` +and `JSON`. Namely, data is loaded into Go values via reflection. + +For the simplest example, consider some TOML file as just a list of keys +and values: + +```toml +Age = 25 +Cats = [ "Cauchy", "Plato" ] +Pi = 3.14 +Perfection = [ 6, 28, 496, 8128 ] +DOB = 1987-07-05T05:45:00Z +``` + +Which could be defined in Go as: + +```go +type Config struct { + Age int + Cats []string + Pi float64 + Perfection []int + DOB time.Time // requires `import time` +} +``` + +And then decoded with: + +```go +var conf Config +if _, err := toml.Decode(tomlData, &conf); err != nil { + // handle error +} +``` + +You can also use struct tags if your struct field name doesn't map to a TOML +key value directly: + +```toml +some_key_NAME = "wat" +``` + +```go +type TOML struct { + ObscureKey string `toml:"some_key_NAME"` +} +``` + +### Using the `encoding.TextUnmarshaler` interface + +Here's an example that automatically parses duration strings into +`time.Duration` values: + +```toml +[[song]] +name = "Thunder Road" +duration = "4m49s" + +[[song]] +name = "Stairway to Heaven" +duration = "8m03s" +``` + +Which can be decoded with: + +```go +type song struct { + Name string + Duration duration +} +type songs struct { + Song []song +} +var favorites songs +if _, err := toml.Decode(blob, &favorites); err != nil { + log.Fatal(err) +} + +for _, s := range favorites.Song { + fmt.Printf("%s (%s)\n", s.Name, s.Duration) +} +``` + +And you'll also need a `duration` type that satisfies the +`encoding.TextUnmarshaler` interface: + +```go +type duration struct { + time.Duration +} + +func (d *duration) UnmarshalText(text []byte) error { + var err error + d.Duration, err = time.ParseDuration(string(text)) + return err +} +``` + +### More complex usage + +Here's an example of how to load the example from the official spec page: + +```toml +# This is a TOML document. Boom. + +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +organization = "GitHub" +bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." +dob = 1979-05-27T07:32:00Z # First class dates? Why not? + +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002 ] +connection_max = 5000 +enabled = true + +[servers] + + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + +[clients] +data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it + +# Line breaks are OK when inside arrays +hosts = [ + "alpha", + "omega" +] +``` + +And the corresponding Go types are: + +```go +type tomlConfig struct { + Title string + Owner ownerInfo + DB database `toml:"database"` + Servers map[string]server + Clients clients +} + +type ownerInfo struct { + Name string + Org string `toml:"organization"` + Bio string + DOB time.Time +} + +type database struct { + Server string + Ports []int + ConnMax int `toml:"connection_max"` + Enabled bool +} + +type server struct { + IP string + DC string +} + +type clients struct { + Data [][]interface{} + Hosts []string +} +``` + +Note that a case insensitive match will be tried if an exact match can't be +found. + +A working example of the above can be found in `_examples/example.{go,toml}`. diff --git a/vendor/github.com/BurntSushi/toml/_examples/example.go b/vendor/github.com/BurntSushi/toml/_examples/example.go new file mode 100644 index 0000000..79f31f2 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/_examples/example.go @@ -0,0 +1,61 @@ +package main + +import ( + "fmt" + "time" + + "github.com/BurntSushi/toml" +) + +type tomlConfig struct { + Title string + Owner ownerInfo + DB database `toml:"database"` + Servers map[string]server + Clients clients +} + +type ownerInfo struct { + Name string + Org string `toml:"organization"` + Bio string + DOB time.Time +} + +type database struct { + Server string + Ports []int + ConnMax int `toml:"connection_max"` + Enabled bool +} + +type server struct { + IP string + DC string +} + +type clients struct { + Data [][]interface{} + Hosts []string +} + +func main() { + var config tomlConfig + if _, err := toml.DecodeFile("example.toml", &config); err != nil { + fmt.Println(err) + return + } + + fmt.Printf("Title: %s\n", config.Title) + fmt.Printf("Owner: %s (%s, %s), Born: %s\n", + config.Owner.Name, config.Owner.Org, config.Owner.Bio, + config.Owner.DOB) + fmt.Printf("Database: %s %v (Max conn. %d), Enabled? %v\n", + config.DB.Server, config.DB.Ports, config.DB.ConnMax, + config.DB.Enabled) + for serverName, server := range config.Servers { + fmt.Printf("Server: %s (%s, %s)\n", serverName, server.IP, server.DC) + } + fmt.Printf("Client data: %v\n", config.Clients.Data) + fmt.Printf("Client hosts: %v\n", config.Clients.Hosts) +} diff --git a/vendor/github.com/BurntSushi/toml/_examples/example.toml b/vendor/github.com/BurntSushi/toml/_examples/example.toml new file mode 100644 index 0000000..32c7a4f --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/_examples/example.toml @@ -0,0 +1,35 @@ +# This is a TOML document. Boom. + +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +organization = "GitHub" +bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." +dob = 1979-05-27T07:32:00Z # First class dates? Why not? + +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002 ] +connection_max = 5000 +enabled = true + +[servers] + + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + +[clients] +data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it + +# Line breaks are OK when inside arrays +hosts = [ + "alpha", + "omega" +] diff --git a/vendor/github.com/BurntSushi/toml/_examples/hard.toml b/vendor/github.com/BurntSushi/toml/_examples/hard.toml new file mode 100644 index 0000000..26145d2 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/_examples/hard.toml @@ -0,0 +1,22 @@ +# Test file for TOML +# Only this one tries to emulate a TOML file written by a user of the kind of parser writers probably hate +# This part you'll really hate + +[the] +test_string = "You'll hate me after this - #" # " Annoying, isn't it? + + [the.hard] + test_array = [ "] ", " # "] # ] There you go, parse this! + test_array2 = [ "Test #11 ]proved that", "Experiment #9 was a success" ] + # You didn't think it'd as easy as chucking out the last #, did you? + another_test_string = " Same thing, but with a string #" + harder_test_string = " And when \"'s are in the string, along with # \"" # "and comments are there too" + # Things will get harder + + [the.hard.bit#] + what? = "You don't think some user won't do that?" + multi_line_array = [ + "]", + # ] Oh yes I did + ] + diff --git a/vendor/github.com/BurntSushi/toml/_examples/implicit.toml b/vendor/github.com/BurntSushi/toml/_examples/implicit.toml new file mode 100644 index 0000000..1dea5ce --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/_examples/implicit.toml @@ -0,0 +1,4 @@ +# [x] you +# [x.y] don't +# [x.y.z] need these +[x.y.z.w] # for this to work diff --git a/vendor/github.com/BurntSushi/toml/_examples/invalid-apples.toml b/vendor/github.com/BurntSushi/toml/_examples/invalid-apples.toml new file mode 100644 index 0000000..74e9e33 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/_examples/invalid-apples.toml @@ -0,0 +1,6 @@ +# DO NOT WANT +[fruit] +type = "apple" + +[fruit.type] +apple = "yes" diff --git a/vendor/github.com/BurntSushi/toml/_examples/invalid.toml b/vendor/github.com/BurntSushi/toml/_examples/invalid.toml new file mode 100644 index 0000000..beb1dba --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/_examples/invalid.toml @@ -0,0 +1,35 @@ +# This is an INVALID TOML document. Boom. +# Can you spot the error without help? + +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +organization = "GitHub" +bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." +dob = 1979-05-27T7:32:00Z # First class dates? Why not? + +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002 ] +connection_max = 5000 +enabled = true + +[servers] + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + +[clients] +data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it + +# Line breaks are OK when inside arrays +hosts = [ + "alpha", + "omega" +] diff --git a/vendor/github.com/BurntSushi/toml/_examples/readme1.toml b/vendor/github.com/BurntSushi/toml/_examples/readme1.toml new file mode 100644 index 0000000..3e1261d --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/_examples/readme1.toml @@ -0,0 +1,5 @@ +Age = 25 +Cats = [ "Cauchy", "Plato" ] +Pi = 3.14 +Perfection = [ 6, 28, 496, 8128 ] +DOB = 1987-07-05T05:45:00Z diff --git a/vendor/github.com/BurntSushi/toml/_examples/readme2.toml b/vendor/github.com/BurntSushi/toml/_examples/readme2.toml new file mode 100644 index 0000000..b51cd93 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/_examples/readme2.toml @@ -0,0 +1 @@ +some_key_NAME = "wat" diff --git a/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING b/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING new file mode 100644 index 0000000..5a8e332 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING @@ -0,0 +1,14 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. + diff --git a/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/README.md b/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/README.md new file mode 100644 index 0000000..93f4e3a --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/README.md @@ -0,0 +1,13 @@ +# Implements the TOML test suite interface + +This is an implementation of the interface expected by +[toml-test](https://github.com/BurntSushi/toml-test) for my +[toml parser written in Go](https://github.com/BurntSushi/toml). +In particular, it maps TOML data on `stdin` to a JSON format on `stdout`. + + +Compatible with TOML version +[v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md) + +Compatible with `toml-test` version +[v0.2.0](https://github.com/BurntSushi/toml-test/tree/v0.2.0) diff --git a/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/main.go b/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/main.go new file mode 100644 index 0000000..14e7557 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/main.go @@ -0,0 +1,90 @@ +// Command toml-test-decoder satisfies the toml-test interface for testing +// TOML decoders. Namely, it accepts TOML on stdin and outputs JSON on stdout. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "log" + "os" + "path" + "time" + + "github.com/BurntSushi/toml" +) + +func init() { + log.SetFlags(0) + + flag.Usage = usage + flag.Parse() +} + +func usage() { + log.Printf("Usage: %s < toml-file\n", path.Base(os.Args[0])) + flag.PrintDefaults() + + os.Exit(1) +} + +func main() { + if flag.NArg() != 0 { + flag.Usage() + } + + var tmp interface{} + if _, err := toml.DecodeReader(os.Stdin, &tmp); err != nil { + log.Fatalf("Error decoding TOML: %s", err) + } + + typedTmp := translate(tmp) + if err := json.NewEncoder(os.Stdout).Encode(typedTmp); err != nil { + log.Fatalf("Error encoding JSON: %s", err) + } +} + +func translate(tomlData interface{}) interface{} { + switch orig := tomlData.(type) { + case map[string]interface{}: + typed := make(map[string]interface{}, len(orig)) + for k, v := range orig { + typed[k] = translate(v) + } + return typed + case []map[string]interface{}: + typed := make([]map[string]interface{}, len(orig)) + for i, v := range orig { + typed[i] = translate(v).(map[string]interface{}) + } + return typed + case []interface{}: + typed := make([]interface{}, len(orig)) + for i, v := range orig { + typed[i] = translate(v) + } + + // We don't really need to tag arrays, but let's be future proof. + // (If TOML ever supports tuples, we'll need this.) + return tag("array", typed) + case time.Time: + return tag("datetime", orig.Format("2006-01-02T15:04:05Z")) + case bool: + return tag("bool", fmt.Sprintf("%v", orig)) + case int64: + return tag("integer", fmt.Sprintf("%d", orig)) + case float64: + return tag("float", fmt.Sprintf("%v", orig)) + case string: + return tag("string", orig) + } + + panic(fmt.Sprintf("Unknown type: %T", tomlData)) +} + +func tag(typeName string, data interface{}) map[string]interface{} { + return map[string]interface{}{ + "type": typeName, + "value": data, + } +} diff --git a/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING b/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING new file mode 100644 index 0000000..5a8e332 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING @@ -0,0 +1,14 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. + diff --git a/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/README.md b/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/README.md new file mode 100644 index 0000000..a45bd4d --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/README.md @@ -0,0 +1,13 @@ +# Implements the TOML test suite interface for TOML encoders + +This is an implementation of the interface expected by +[toml-test](https://github.com/BurntSushi/toml-test) for the +[TOML encoder](https://github.com/BurntSushi/toml). +In particular, it maps JSON data on `stdin` to a TOML format on `stdout`. + + +Compatible with TOML version +[v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md) + +Compatible with `toml-test` version +[v0.2.0](https://github.com/BurntSushi/toml-test/tree/v0.2.0) diff --git a/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/main.go b/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/main.go new file mode 100644 index 0000000..092cc68 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/main.go @@ -0,0 +1,131 @@ +// Command toml-test-encoder satisfies the toml-test interface for testing +// TOML encoders. Namely, it accepts JSON on stdin and outputs TOML on stdout. +package main + +import ( + "encoding/json" + "flag" + "log" + "os" + "path" + "strconv" + "time" + + "github.com/BurntSushi/toml" +) + +func init() { + log.SetFlags(0) + + flag.Usage = usage + flag.Parse() +} + +func usage() { + log.Printf("Usage: %s < json-file\n", path.Base(os.Args[0])) + flag.PrintDefaults() + + os.Exit(1) +} + +func main() { + if flag.NArg() != 0 { + flag.Usage() + } + + var tmp interface{} + if err := json.NewDecoder(os.Stdin).Decode(&tmp); err != nil { + log.Fatalf("Error decoding JSON: %s", err) + } + + tomlData := translate(tmp) + if err := toml.NewEncoder(os.Stdout).Encode(tomlData); err != nil { + log.Fatalf("Error encoding TOML: %s", err) + } +} + +func translate(typedJson interface{}) interface{} { + switch v := typedJson.(type) { + case map[string]interface{}: + if len(v) == 2 && in("type", v) && in("value", v) { + return untag(v) + } + m := make(map[string]interface{}, len(v)) + for k, v2 := range v { + m[k] = translate(v2) + } + return m + case []interface{}: + tabArray := make([]map[string]interface{}, len(v)) + for i := range v { + if m, ok := translate(v[i]).(map[string]interface{}); ok { + tabArray[i] = m + } else { + log.Fatalf("JSON arrays may only contain objects. This " + + "corresponds to only tables being allowed in " + + "TOML table arrays.") + } + } + return tabArray + } + log.Fatalf("Unrecognized JSON format '%T'.", typedJson) + panic("unreachable") +} + +func untag(typed map[string]interface{}) interface{} { + t := typed["type"].(string) + v := typed["value"] + switch t { + case "string": + return v.(string) + case "integer": + v := v.(string) + n, err := strconv.Atoi(v) + if err != nil { + log.Fatalf("Could not parse '%s' as integer: %s", v, err) + } + return n + case "float": + v := v.(string) + f, err := strconv.ParseFloat(v, 64) + if err != nil { + log.Fatalf("Could not parse '%s' as float64: %s", v, err) + } + return f + case "datetime": + v := v.(string) + t, err := time.Parse("2006-01-02T15:04:05Z", v) + if err != nil { + log.Fatalf("Could not parse '%s' as a datetime: %s", v, err) + } + return t + case "bool": + v := v.(string) + switch v { + case "true": + return true + case "false": + return false + } + log.Fatalf("Could not parse '%s' as a boolean.", v) + case "array": + v := v.([]interface{}) + array := make([]interface{}, len(v)) + for i := range v { + if m, ok := v[i].(map[string]interface{}); ok { + array[i] = untag(m) + } else { + log.Fatalf("Arrays may only contain other arrays or "+ + "primitive values, but found a '%T'.", m) + } + } + return array + } + log.Fatalf("Unrecognized tag type '%s'.", t) + panic("unreachable") +} + +func in(key string, m map[string]interface{}) bool { + _, ok := m[key] + return ok +} diff --git a/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING b/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING new file mode 100644 index 0000000..5a8e332 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING @@ -0,0 +1,14 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. + diff --git a/vendor/github.com/BurntSushi/toml/cmd/tomlv/README.md b/vendor/github.com/BurntSushi/toml/cmd/tomlv/README.md new file mode 100644 index 0000000..51231e2 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/cmd/tomlv/README.md @@ -0,0 +1,21 @@ +# TOML Validator + +If Go is installed, it's simple to try it out: + +```bash +go get github.com/BurntSushi/toml/cmd/tomlv +tomlv some-toml-file.toml +``` + +You can see the types of every key in a TOML file with: + +```bash +tomlv -types some-toml-file.toml +``` + +At the moment, only one error message is reported at a time. Error messages +include line numbers. No output means that the files given are valid TOML, or +there is a bug in `tomlv`. + +Compatible with TOML version +[v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md) diff --git a/vendor/github.com/BurntSushi/toml/cmd/tomlv/main.go b/vendor/github.com/BurntSushi/toml/cmd/tomlv/main.go new file mode 100644 index 0000000..c7d689a --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/cmd/tomlv/main.go @@ -0,0 +1,61 @@ +// Command tomlv validates TOML documents and prints each key's type. +package main + +import ( + "flag" + "fmt" + "log" + "os" + "path" + "strings" + "text/tabwriter" + + "github.com/BurntSushi/toml" +) + +var ( + flagTypes = false +) + +func init() { + log.SetFlags(0) + + flag.BoolVar(&flagTypes, "types", flagTypes, + "When set, the types of every defined key will be shown.") + + flag.Usage = usage + flag.Parse() +} + +func usage() { + log.Printf("Usage: %s toml-file [ toml-file ... ]\n", + path.Base(os.Args[0])) + flag.PrintDefaults() + + os.Exit(1) +} + +func main() { + if flag.NArg() < 1 { + flag.Usage() + } + for _, f := range flag.Args() { + var tmp interface{} + md, err := toml.DecodeFile(f, &tmp) + if err != nil { + log.Fatalf("Error in '%s': %s", f, err) + } + if flagTypes { + printTypes(md) + } + } +} + +func printTypes(md toml.MetaData) { + tabw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + for _, key := range md.Keys() { + fmt.Fprintf(tabw, "%s%s\t%s\n", + strings.Repeat(" ", len(key)-1), key, md.Type(key...)) + } + tabw.Flush() +} diff --git a/vendor/github.com/BurntSushi/toml/decode.go b/vendor/github.com/BurntSushi/toml/decode.go new file mode 100644 index 0000000..b0fd51d --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/decode.go @@ -0,0 +1,509 @@ +package toml + +import ( + "fmt" + "io" + "io/ioutil" + "math" + "reflect" + "strings" + "time" +) + +func e(format string, args ...interface{}) error { + return fmt.Errorf("toml: "+format, args...) +} + +// Unmarshaler is the interface implemented by objects that can unmarshal a +// TOML description of themselves. +type Unmarshaler interface { + UnmarshalTOML(interface{}) error +} + +// Unmarshal decodes the contents of `p` in TOML format into a pointer `v`. +func Unmarshal(p []byte, v interface{}) error { + _, err := Decode(string(p), v) + return err +} + +// Primitive is a TOML value that hasn't been decoded into a Go value. +// When using the various `Decode*` functions, the type `Primitive` may +// be given to any value, and its decoding will be delayed. +// +// A `Primitive` value can be decoded using the `PrimitiveDecode` function. +// +// The underlying representation of a `Primitive` value is subject to change. +// Do not rely on it. +// +// N.B. Primitive values are still parsed, so using them will only avoid +// the overhead of reflection. They can be useful when you don't know the +// exact type of TOML data until run time. +type Primitive struct { + undecoded interface{} + context Key +} + +// DEPRECATED! +// +// Use MetaData.PrimitiveDecode instead. +func PrimitiveDecode(primValue Primitive, v interface{}) error { + md := MetaData{decoded: make(map[string]bool)} + return md.unify(primValue.undecoded, rvalue(v)) +} + +// PrimitiveDecode is just like the other `Decode*` functions, except it +// decodes a TOML value that has already been parsed. Valid primitive values +// can *only* be obtained from values filled by the decoder functions, +// including this method. (i.e., `v` may contain more `Primitive` +// values.) +// +// Meta data for primitive values is included in the meta data returned by +// the `Decode*` functions with one exception: keys returned by the Undecoded +// method will only reflect keys that were decoded. Namely, any keys hidden +// behind a Primitive will be considered undecoded. Executing this method will +// update the undecoded keys in the meta data. (See the example.) +func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error { + md.context = primValue.context + defer func() { md.context = nil }() + return md.unify(primValue.undecoded, rvalue(v)) +} + +// Decode will decode the contents of `data` in TOML format into a pointer +// `v`. +// +// TOML hashes correspond to Go structs or maps. (Dealer's choice. They can be +// used interchangeably.) +// +// TOML arrays of tables correspond to either a slice of structs or a slice +// of maps. +// +// TOML datetimes correspond to Go `time.Time` values. +// +// All other TOML types (float, string, int, bool and array) correspond +// to the obvious Go types. +// +// An exception to the above rules is if a type implements the +// encoding.TextUnmarshaler interface. In this case, any primitive TOML value +// (floats, strings, integers, booleans and datetimes) will be converted to +// a byte string and given to the value's UnmarshalText method. See the +// Unmarshaler example for a demonstration with time duration strings. +// +// Key mapping +// +// TOML keys can map to either keys in a Go map or field names in a Go +// struct. The special `toml` struct tag may be used to map TOML keys to +// struct fields that don't match the key name exactly. (See the example.) +// A case insensitive match to struct names will be tried if an exact match +// can't be found. +// +// The mapping between TOML values and Go values is loose. That is, there +// may exist TOML values that cannot be placed into your representation, and +// there may be parts of your representation that do not correspond to +// TOML values. This loose mapping can be made stricter by using the IsDefined +// and/or Undecoded methods on the MetaData returned. +// +// This decoder will not handle cyclic types. If a cyclic type is passed, +// `Decode` will not terminate. +func Decode(data string, v interface{}) (MetaData, error) { + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr { + return MetaData{}, e("Decode of non-pointer %s", reflect.TypeOf(v)) + } + if rv.IsNil() { + return MetaData{}, e("Decode of nil %s", reflect.TypeOf(v)) + } + p, err := parse(data) + if err != nil { + return MetaData{}, err + } + md := MetaData{ + p.mapping, p.types, p.ordered, + make(map[string]bool, len(p.ordered)), nil, + } + return md, md.unify(p.mapping, indirect(rv)) +} + +// DecodeFile is just like Decode, except it will automatically read the +// contents of the file at `fpath` and decode it for you. +func DecodeFile(fpath string, v interface{}) (MetaData, error) { + bs, err := ioutil.ReadFile(fpath) + if err != nil { + return MetaData{}, err + } + return Decode(string(bs), v) +} + +// DecodeReader is just like Decode, except it will consume all bytes +// from the reader and decode it for you. +func DecodeReader(r io.Reader, v interface{}) (MetaData, error) { + bs, err := ioutil.ReadAll(r) + if err != nil { + return MetaData{}, err + } + return Decode(string(bs), v) +} + +// unify performs a sort of type unification based on the structure of `rv`, +// which is the client representation. +// +// Any type mismatch produces an error. Finding a type that we don't know +// how to handle produces an unsupported type error. +func (md *MetaData) unify(data interface{}, rv reflect.Value) error { + + // Special case. Look for a `Primitive` value. + if rv.Type() == reflect.TypeOf((*Primitive)(nil)).Elem() { + // Save the undecoded data and the key context into the primitive + // value. + context := make(Key, len(md.context)) + copy(context, md.context) + rv.Set(reflect.ValueOf(Primitive{ + undecoded: data, + context: context, + })) + return nil + } + + // Special case. Unmarshaler Interface support. + if rv.CanAddr() { + if v, ok := rv.Addr().Interface().(Unmarshaler); ok { + return v.UnmarshalTOML(data) + } + } + + // Special case. Handle time.Time values specifically. + // TODO: Remove this code when we decide to drop support for Go 1.1. + // This isn't necessary in Go 1.2 because time.Time satisfies the encoding + // interfaces. + if rv.Type().AssignableTo(rvalue(time.Time{}).Type()) { + return md.unifyDatetime(data, rv) + } + + // Special case. Look for a value satisfying the TextUnmarshaler interface. + if v, ok := rv.Interface().(TextUnmarshaler); ok { + return md.unifyText(data, v) + } + // BUG(burntsushi) + // The behavior here is incorrect whenever a Go type satisfies the + // encoding.TextUnmarshaler interface but also corresponds to a TOML + // hash or array. In particular, the unmarshaler should only be applied + // to primitive TOML values. But at this point, it will be applied to + // all kinds of values and produce an incorrect error whenever those values + // are hashes or arrays (including arrays of tables). + + k := rv.Kind() + + // laziness + if k >= reflect.Int && k <= reflect.Uint64 { + return md.unifyInt(data, rv) + } + switch k { + case reflect.Ptr: + elem := reflect.New(rv.Type().Elem()) + err := md.unify(data, reflect.Indirect(elem)) + if err != nil { + return err + } + rv.Set(elem) + return nil + case reflect.Struct: + return md.unifyStruct(data, rv) + case reflect.Map: + return md.unifyMap(data, rv) + case reflect.Array: + return md.unifyArray(data, rv) + case reflect.Slice: + return md.unifySlice(data, rv) + case reflect.String: + return md.unifyString(data, rv) + case reflect.Bool: + return md.unifyBool(data, rv) + case reflect.Interface: + // we only support empty interfaces. + if rv.NumMethod() > 0 { + return e("unsupported type %s", rv.Type()) + } + return md.unifyAnything(data, rv) + case reflect.Float32: + fallthrough + case reflect.Float64: + return md.unifyFloat64(data, rv) + } + return e("unsupported type %s", rv.Kind()) +} + +func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { + tmap, ok := mapping.(map[string]interface{}) + if !ok { + if mapping == nil { + return nil + } + return e("type mismatch for %s: expected table but found %T", + rv.Type().String(), mapping) + } + + for key, datum := range tmap { + var f *field + fields := cachedTypeFields(rv.Type()) + for i := range fields { + ff := &fields[i] + if ff.name == key { + f = ff + break + } + if f == nil && strings.EqualFold(ff.name, key) { + f = ff + } + } + if f != nil { + subv := rv + for _, i := range f.index { + subv = indirect(subv.Field(i)) + } + if isUnifiable(subv) { + md.decoded[md.context.add(key).String()] = true + md.context = append(md.context, key) + if err := md.unify(datum, subv); err != nil { + return err + } + md.context = md.context[0 : len(md.context)-1] + } else if f.name != "" { + // Bad user! No soup for you! + return e("cannot write unexported field %s.%s", + rv.Type().String(), f.name) + } + } + } + return nil +} + +func (md *MetaData) unifyMap(mapping interface{}, rv reflect.Value) error { + tmap, ok := mapping.(map[string]interface{}) + if !ok { + if tmap == nil { + return nil + } + return badtype("map", mapping) + } + if rv.IsNil() { + rv.Set(reflect.MakeMap(rv.Type())) + } + for k, v := range tmap { + md.decoded[md.context.add(k).String()] = true + md.context = append(md.context, k) + + rvkey := indirect(reflect.New(rv.Type().Key())) + rvval := reflect.Indirect(reflect.New(rv.Type().Elem())) + if err := md.unify(v, rvval); err != nil { + return err + } + md.context = md.context[0 : len(md.context)-1] + + rvkey.SetString(k) + rv.SetMapIndex(rvkey, rvval) + } + return nil +} + +func (md *MetaData) unifyArray(data interface{}, rv reflect.Value) error { + datav := reflect.ValueOf(data) + if datav.Kind() != reflect.Slice { + if !datav.IsValid() { + return nil + } + return badtype("slice", data) + } + sliceLen := datav.Len() + if sliceLen != rv.Len() { + return e("expected array length %d; got TOML array of length %d", + rv.Len(), sliceLen) + } + return md.unifySliceArray(datav, rv) +} + +func (md *MetaData) unifySlice(data interface{}, rv reflect.Value) error { + datav := reflect.ValueOf(data) + if datav.Kind() != reflect.Slice { + if !datav.IsValid() { + return nil + } + return badtype("slice", data) + } + n := datav.Len() + if rv.IsNil() || rv.Cap() < n { + rv.Set(reflect.MakeSlice(rv.Type(), n, n)) + } + rv.SetLen(n) + return md.unifySliceArray(datav, rv) +} + +func (md *MetaData) unifySliceArray(data, rv reflect.Value) error { + sliceLen := data.Len() + for i := 0; i < sliceLen; i++ { + v := data.Index(i).Interface() + sliceval := indirect(rv.Index(i)) + if err := md.unify(v, sliceval); err != nil { + return err + } + } + return nil +} + +func (md *MetaData) unifyDatetime(data interface{}, rv reflect.Value) error { + if _, ok := data.(time.Time); ok { + rv.Set(reflect.ValueOf(data)) + return nil + } + return badtype("time.Time", data) +} + +func (md *MetaData) unifyString(data interface{}, rv reflect.Value) error { + if s, ok := data.(string); ok { + rv.SetString(s) + return nil + } + return badtype("string", data) +} + +func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error { + if num, ok := data.(float64); ok { + switch rv.Kind() { + case reflect.Float32: + fallthrough + case reflect.Float64: + rv.SetFloat(num) + default: + panic("bug") + } + return nil + } + return badtype("float", data) +} + +func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { + if num, ok := data.(int64); ok { + if rv.Kind() >= reflect.Int && rv.Kind() <= reflect.Int64 { + switch rv.Kind() { + case reflect.Int, reflect.Int64: + // No bounds checking necessary. + case reflect.Int8: + if num < math.MinInt8 || num > math.MaxInt8 { + return e("value %d is out of range for int8", num) + } + case reflect.Int16: + if num < math.MinInt16 || num > math.MaxInt16 { + return e("value %d is out of range for int16", num) + } + case reflect.Int32: + if num < math.MinInt32 || num > math.MaxInt32 { + return e("value %d is out of range for int32", num) + } + } + rv.SetInt(num) + } else if rv.Kind() >= reflect.Uint && rv.Kind() <= reflect.Uint64 { + unum := uint64(num) + switch rv.Kind() { + case reflect.Uint, reflect.Uint64: + // No bounds checking necessary. + case reflect.Uint8: + if num < 0 || unum > math.MaxUint8 { + return e("value %d is out of range for uint8", num) + } + case reflect.Uint16: + if num < 0 || unum > math.MaxUint16 { + return e("value %d is out of range for uint16", num) + } + case reflect.Uint32: + if num < 0 || unum > math.MaxUint32 { + return e("value %d is out of range for uint32", num) + } + } + rv.SetUint(unum) + } else { + panic("unreachable") + } + return nil + } + return badtype("integer", data) +} + +func (md *MetaData) unifyBool(data interface{}, rv reflect.Value) error { + if b, ok := data.(bool); ok { + rv.SetBool(b) + return nil + } + return badtype("boolean", data) +} + +func (md *MetaData) unifyAnything(data interface{}, rv reflect.Value) error { + rv.Set(reflect.ValueOf(data)) + return nil +} + +func (md *MetaData) unifyText(data interface{}, v TextUnmarshaler) error { + var s string + switch sdata := data.(type) { + case TextMarshaler: + text, err := sdata.MarshalText() + if err != nil { + return err + } + s = string(text) + case fmt.Stringer: + s = sdata.String() + case string: + s = sdata + case bool: + s = fmt.Sprintf("%v", sdata) + case int64: + s = fmt.Sprintf("%d", sdata) + case float64: + s = fmt.Sprintf("%f", sdata) + default: + return badtype("primitive (string-like)", data) + } + if err := v.UnmarshalText([]byte(s)); err != nil { + return err + } + return nil +} + +// rvalue returns a reflect.Value of `v`. All pointers are resolved. +func rvalue(v interface{}) reflect.Value { + return indirect(reflect.ValueOf(v)) +} + +// indirect returns the value pointed to by a pointer. +// Pointers are followed until the value is not a pointer. +// New values are allocated for each nil pointer. +// +// An exception to this rule is if the value satisfies an interface of +// interest to us (like encoding.TextUnmarshaler). +func indirect(v reflect.Value) reflect.Value { + if v.Kind() != reflect.Ptr { + if v.CanSet() { + pv := v.Addr() + if _, ok := pv.Interface().(TextUnmarshaler); ok { + return pv + } + } + return v + } + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + return indirect(reflect.Indirect(v)) +} + +func isUnifiable(rv reflect.Value) bool { + if rv.CanSet() { + return true + } + if _, ok := rv.Interface().(TextUnmarshaler); ok { + return true + } + return false +} + +func badtype(expected string, data interface{}) error { + return e("cannot load TOML value of type %T into a Go %s", data, expected) +} diff --git a/vendor/github.com/BurntSushi/toml/decode_meta.go b/vendor/github.com/BurntSushi/toml/decode_meta.go new file mode 100644 index 0000000..b9914a6 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/decode_meta.go @@ -0,0 +1,121 @@ +package toml + +import "strings" + +// MetaData allows access to meta information about TOML data that may not +// be inferrable via reflection. In particular, whether a key has been defined +// and the TOML type of a key. +type MetaData struct { + mapping map[string]interface{} + types map[string]tomlType + keys []Key + decoded map[string]bool + context Key // Used only during decoding. +} + +// IsDefined returns true if the key given exists in the TOML data. The key +// should be specified hierarchially. e.g., +// +// // access the TOML key 'a.b.c' +// IsDefined("a", "b", "c") +// +// IsDefined will return false if an empty key given. Keys are case sensitive. +func (md *MetaData) IsDefined(key ...string) bool { + if len(key) == 0 { + return false + } + + var hash map[string]interface{} + var ok bool + var hashOrVal interface{} = md.mapping + for _, k := range key { + if hash, ok = hashOrVal.(map[string]interface{}); !ok { + return false + } + if hashOrVal, ok = hash[k]; !ok { + return false + } + } + return true +} + +// Type returns a string representation of the type of the key specified. +// +// Type will return the empty string if given an empty key or a key that +// does not exist. Keys are case sensitive. +func (md *MetaData) Type(key ...string) string { + fullkey := strings.Join(key, ".") + if typ, ok := md.types[fullkey]; ok { + return typ.typeString() + } + return "" +} + +// Key is the type of any TOML key, including key groups. Use (MetaData).Keys +// to get values of this type. +type Key []string + +func (k Key) String() string { + return strings.Join(k, ".") +} + +func (k Key) maybeQuotedAll() string { + var ss []string + for i := range k { + ss = append(ss, k.maybeQuoted(i)) + } + return strings.Join(ss, ".") +} + +func (k Key) maybeQuoted(i int) string { + quote := false + for _, c := range k[i] { + if !isBareKeyChar(c) { + quote = true + break + } + } + if quote { + return "\"" + strings.Replace(k[i], "\"", "\\\"", -1) + "\"" + } + return k[i] +} + +func (k Key) add(piece string) Key { + newKey := make(Key, len(k)+1) + copy(newKey, k) + newKey[len(k)] = piece + return newKey +} + +// Keys returns a slice of every key in the TOML data, including key groups. +// Each key is itself a slice, where the first element is the top of the +// hierarchy and the last is the most specific. +// +// The list will have the same order as the keys appeared in the TOML data. +// +// All keys returned are non-empty. +func (md *MetaData) Keys() []Key { + return md.keys +} + +// Undecoded returns all keys that have not been decoded in the order in which +// they appear in the original TOML document. +// +// This includes keys that haven't been decoded because of a Primitive value. +// Once the Primitive value is decoded, the keys will be considered decoded. +// +// Also note that decoding into an empty interface will result in no decoding, +// and so no keys will be considered decoded. +// +// In this sense, the Undecoded keys correspond to keys in the TOML document +// that do not have a concrete type in your representation. +func (md *MetaData) Undecoded() []Key { + undecoded := make([]Key, 0, len(md.keys)) + for _, key := range md.keys { + if !md.decoded[key.String()] { + undecoded = append(undecoded, key) + } + } + return undecoded +} diff --git a/vendor/github.com/BurntSushi/toml/decode_test.go b/vendor/github.com/BurntSushi/toml/decode_test.go new file mode 100644 index 0000000..95bc987 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/decode_test.go @@ -0,0 +1,1461 @@ +package toml + +import ( + "fmt" + "log" + "math" + "reflect" + "strings" + "testing" + "time" +) + +func TestDecodeSimple(t *testing.T) { + var testSimple = ` +age = 250 +andrew = "gallant" +kait = "brady" +now = 1987-07-05T05:45:00Z +nowEast = 2017-06-22T16:15:21+08:00 +nowWest = 2017-06-22T02:14:36-06:00 +yesOrNo = true +pi = 3.14 +colors = [ + ["red", "green", "blue"], + ["cyan", "magenta", "yellow", "black"], +] + +[My.Cats] +plato = "cat 1" +cauchy = "cat 2" +` + + type cats struct { + Plato string + Cauchy string + } + type simple struct { + Age int + Colors [][]string + Pi float64 + YesOrNo bool + Now time.Time + NowEast time.Time + NowWest time.Time + Andrew string + Kait string + My map[string]cats + } + + var val simple + _, err := Decode(testSimple, &val) + if err != nil { + t.Fatal(err) + } + + now, err := time.Parse("2006-01-02T15:04:05", "1987-07-05T05:45:00") + if err != nil { + panic(err) + } + nowEast, err := time.Parse("2006-01-02T15:04:05-07:00", "2017-06-22T16:15:21+08:00") + if err != nil { + panic(err) + } + nowWest, err := time.Parse("2006-01-02T15:04:05-07:00", "2017-06-22T02:14:36-06:00") + if err != nil { + panic(err) + } + var answer = simple{ + Age: 250, + Andrew: "gallant", + Kait: "brady", + Now: now, + NowEast: nowEast, + NowWest: nowWest, + YesOrNo: true, + Pi: 3.14, + Colors: [][]string{ + {"red", "green", "blue"}, + {"cyan", "magenta", "yellow", "black"}, + }, + My: map[string]cats{ + "Cats": {Plato: "cat 1", Cauchy: "cat 2"}, + }, + } + if !reflect.DeepEqual(val, answer) { + t.Fatalf("Expected\n-----\n%#v\n-----\nbut got\n-----\n%#v\n", + answer, val) + } +} + +func TestDecodeEmbedded(t *testing.T) { + type Dog struct{ Name string } + type Age int + type cat struct{ Name string } + + for _, test := range []struct { + label string + input string + decodeInto interface{} + wantDecoded interface{} + }{ + { + label: "embedded struct", + input: `Name = "milton"`, + decodeInto: &struct{ Dog }{}, + wantDecoded: &struct{ Dog }{Dog{"milton"}}, + }, + { + label: "embedded non-nil pointer to struct", + input: `Name = "milton"`, + decodeInto: &struct{ *Dog }{}, + wantDecoded: &struct{ *Dog }{&Dog{"milton"}}, + }, + { + label: "embedded nil pointer to struct", + input: ``, + decodeInto: &struct{ *Dog }{}, + wantDecoded: &struct{ *Dog }{nil}, + }, + { + label: "unexported embedded struct", + input: `Name = "socks"`, + decodeInto: &struct{ cat }{}, + wantDecoded: &struct{ cat }{cat{"socks"}}, + }, + { + label: "embedded int", + input: `Age = -5`, + decodeInto: &struct{ Age }{}, + wantDecoded: &struct{ Age }{-5}, + }, + } { + _, err := Decode(test.input, test.decodeInto) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(test.wantDecoded, test.decodeInto) { + t.Errorf("%s: want decoded == %+v, got %+v", + test.label, test.wantDecoded, test.decodeInto) + } + } +} + +func TestDecodeIgnoredFields(t *testing.T) { + type simple struct { + Number int `toml:"-"` + } + const input = ` +Number = 123 +- = 234 +` + var s simple + if _, err := Decode(input, &s); err != nil { + t.Fatal(err) + } + if s.Number != 0 { + t.Errorf("got: %d; want 0", s.Number) + } +} + +func TestTableArrays(t *testing.T) { + var tomlTableArrays = ` +[[albums]] +name = "Born to Run" + + [[albums.songs]] + name = "Jungleland" + + [[albums.songs]] + name = "Meeting Across the River" + +[[albums]] +name = "Born in the USA" + + [[albums.songs]] + name = "Glory Days" + + [[albums.songs]] + name = "Dancing in the Dark" +` + + type Song struct { + Name string + } + + type Album struct { + Name string + Songs []Song + } + + type Music struct { + Albums []Album + } + + expected := Music{[]Album{ + {"Born to Run", []Song{{"Jungleland"}, {"Meeting Across the River"}}}, + {"Born in the USA", []Song{{"Glory Days"}, {"Dancing in the Dark"}}}, + }} + var got Music + if _, err := Decode(tomlTableArrays, &got); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(expected, got) { + t.Fatalf("\n%#v\n!=\n%#v\n", expected, got) + } +} + +func TestTableNesting(t *testing.T) { + for _, tt := range []struct { + t string + want []string + }{ + {"[a.b.c]", []string{"a", "b", "c"}}, + {`[a."b.c"]`, []string{"a", "b.c"}}, + {`[a.'b.c']`, []string{"a", "b.c"}}, + {`[a.' b ']`, []string{"a", " b "}}, + {"[ d.e.f ]", []string{"d", "e", "f"}}, + {"[ g . h . i ]", []string{"g", "h", "i"}}, + {`[ j . "ʞ" . 'l' ]`, []string{"j", "ʞ", "l"}}, + } { + var m map[string]interface{} + if _, err := Decode(tt.t, &m); err != nil { + t.Errorf("Decode(%q): got error: %s", tt.t, err) + continue + } + if keys := extractNestedKeys(m); !reflect.DeepEqual(keys, tt.want) { + t.Errorf("Decode(%q): got nested keys %#v; want %#v", + tt.t, keys, tt.want) + } + } +} + +func extractNestedKeys(v map[string]interface{}) []string { + var result []string + for { + if len(v) != 1 { + return result + } + for k, m := range v { + result = append(result, k) + var ok bool + v, ok = m.(map[string]interface{}) + if !ok { + return result + } + } + + } +} + +// Case insensitive matching tests. +// A bit more comprehensive than needed given the current implementation, +// but implementations change. +// Probably still missing demonstrations of some ugly corner cases regarding +// case insensitive matching and multiple fields. +func TestCase(t *testing.T) { + var caseToml = ` +tOpString = "string" +tOpInt = 1 +tOpFloat = 1.1 +tOpBool = true +tOpdate = 2006-01-02T15:04:05Z +tOparray = [ "array" ] +Match = "i should be in Match only" +MatcH = "i should be in MatcH only" +once = "just once" +[nEst.eD] +nEstedString = "another string" +` + + type InsensitiveEd struct { + NestedString string + } + + type InsensitiveNest struct { + Ed InsensitiveEd + } + + type Insensitive struct { + TopString string + TopInt int + TopFloat float64 + TopBool bool + TopDate time.Time + TopArray []string + Match string + MatcH string + Once string + OncE string + Nest InsensitiveNest + } + + tme, err := time.Parse(time.RFC3339, time.RFC3339[:len(time.RFC3339)-5]) + if err != nil { + panic(err) + } + expected := Insensitive{ + TopString: "string", + TopInt: 1, + TopFloat: 1.1, + TopBool: true, + TopDate: tme, + TopArray: []string{"array"}, + MatcH: "i should be in MatcH only", + Match: "i should be in Match only", + Once: "just once", + OncE: "", + Nest: InsensitiveNest{ + Ed: InsensitiveEd{NestedString: "another string"}, + }, + } + var got Insensitive + if _, err := Decode(caseToml, &got); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(expected, got) { + t.Fatalf("\n%#v\n!=\n%#v\n", expected, got) + } +} + +func TestPointers(t *testing.T) { + type Object struct { + Type string + Description string + } + + type Dict struct { + NamedObject map[string]*Object + BaseObject *Object + Strptr *string + Strptrs []*string + } + s1, s2, s3 := "blah", "abc", "def" + expected := &Dict{ + Strptr: &s1, + Strptrs: []*string{&s2, &s3}, + NamedObject: map[string]*Object{ + "foo": {"FOO", "fooooo!!!"}, + "bar": {"BAR", "ba-ba-ba-ba-barrrr!!!"}, + }, + BaseObject: &Object{"BASE", "da base"}, + } + + ex1 := ` +Strptr = "blah" +Strptrs = ["abc", "def"] + +[NamedObject.foo] +Type = "FOO" +Description = "fooooo!!!" + +[NamedObject.bar] +Type = "BAR" +Description = "ba-ba-ba-ba-barrrr!!!" + +[BaseObject] +Type = "BASE" +Description = "da base" +` + dict := new(Dict) + _, err := Decode(ex1, dict) + if err != nil { + t.Errorf("Decode error: %v", err) + } + if !reflect.DeepEqual(expected, dict) { + t.Fatalf("\n%#v\n!=\n%#v\n", expected, dict) + } +} + +func TestDecodeDatetime(t *testing.T) { + const noTimestamp = "2006-01-02T15:04:05" + for _, tt := range []struct { + s string + t string + format string + }{ + {"1979-05-27T07:32:00Z", "1979-05-27T07:32:00Z", time.RFC3339}, + {"1979-05-27T00:32:00-07:00", "1979-05-27T00:32:00-07:00", time.RFC3339}, + { + "1979-05-27T00:32:00.999999-07:00", + "1979-05-27T00:32:00.999999-07:00", + time.RFC3339, + }, + {"1979-05-27T07:32:00", "1979-05-27T07:32:00", noTimestamp}, + { + "1979-05-27T00:32:00.999999", + "1979-05-27T00:32:00.999999", + noTimestamp, + }, + {"1979-05-27", "1979-05-27T00:00:00", noTimestamp}, + } { + var x struct{ D time.Time } + input := "d = " + tt.s + if _, err := Decode(input, &x); err != nil { + t.Errorf("Decode(%q): got error: %s", input, err) + continue + } + want, err := time.ParseInLocation(tt.format, tt.t, time.Local) + if err != nil { + panic(err) + } + if !x.D.Equal(want) { + t.Errorf("Decode(%q): got %s; want %s", input, x.D, want) + } + } +} + +func TestDecodeBadDatetime(t *testing.T) { + var x struct{ T time.Time } + for _, s := range []string{ + "123", + "2006-01-50T00:00:00Z", + "2006-01-30T00:00", + "2006-01-30T", + } { + input := "T = " + s + if _, err := Decode(input, &x); err == nil { + t.Errorf("Expected invalid DateTime error for %q", s) + } + } +} + +func TestDecodeMultilineStrings(t *testing.T) { + var x struct { + S string + } + const s0 = `s = """ +a b \n c +d e f +"""` + if _, err := Decode(s0, &x); err != nil { + t.Fatal(err) + } + if want := "a b \n c\nd e f\n"; x.S != want { + t.Errorf("got: %q; want: %q", x.S, want) + } + const s1 = `s = """a b c\ +"""` + if _, err := Decode(s1, &x); err != nil { + t.Fatal(err) + } + if want := "a b c"; x.S != want { + t.Errorf("got: %q; want: %q", x.S, want) + } +} + +type sphere struct { + Center [3]float64 + Radius float64 +} + +func TestDecodeSimpleArray(t *testing.T) { + var s1 sphere + if _, err := Decode(`center = [0.0, 1.5, 0.0]`, &s1); err != nil { + t.Fatal(err) + } +} + +func TestDecodeArrayWrongSize(t *testing.T) { + var s1 sphere + if _, err := Decode(`center = [0.1, 2.3]`, &s1); err == nil { + t.Fatal("Expected array type mismatch error") + } +} + +func TestDecodeLargeIntoSmallInt(t *testing.T) { + type table struct { + Value int8 + } + var tab table + if _, err := Decode(`value = 500`, &tab); err == nil { + t.Fatal("Expected integer out-of-bounds error.") + } +} + +func TestDecodeSizedInts(t *testing.T) { + type table struct { + U8 uint8 + U16 uint16 + U32 uint32 + U64 uint64 + U uint + I8 int8 + I16 int16 + I32 int32 + I64 int64 + I int + } + answer := table{1, 1, 1, 1, 1, -1, -1, -1, -1, -1} + toml := ` + u8 = 1 + u16 = 1 + u32 = 1 + u64 = 1 + u = 1 + i8 = -1 + i16 = -1 + i32 = -1 + i64 = -1 + i = -1 + ` + var tab table + if _, err := Decode(toml, &tab); err != nil { + t.Fatal(err.Error()) + } + if answer != tab { + t.Fatalf("Expected %#v but got %#v", answer, tab) + } +} + +func TestDecodeInts(t *testing.T) { + for _, tt := range []struct { + s string + want int64 + }{ + {"0", 0}, + {"+99", 99}, + {"-10", -10}, + {"1_234_567", 1234567}, + {"1_2_3_4", 1234}, + {"-9_223_372_036_854_775_808", math.MinInt64}, + {"9_223_372_036_854_775_807", math.MaxInt64}, + } { + var x struct{ N int64 } + input := "n = " + tt.s + if _, err := Decode(input, &x); err != nil { + t.Errorf("Decode(%q): got error: %s", input, err) + continue + } + if x.N != tt.want { + t.Errorf("Decode(%q): got %d; want %d", input, x.N, tt.want) + } + } +} + +func TestDecodeFloats(t *testing.T) { + for _, tt := range []struct { + s string + want float64 + }{ + {"+1.0", 1}, + {"3.1415", 3.1415}, + {"-0.01", -0.01}, + {"5e+22", 5e22}, + {"1e6", 1e6}, + {"-2E-2", -2e-2}, + {"6.626e-34", 6.626e-34}, + {"9_224_617.445_991_228_313", 9224617.445991228313}, + {"9_876.54_32e1_0", 9876.5432e10}, + } { + var x struct{ N float64 } + input := "n = " + tt.s + if _, err := Decode(input, &x); err != nil { + t.Errorf("Decode(%q): got error: %s", input, err) + continue + } + if x.N != tt.want { + t.Errorf("Decode(%q): got %f; want %f", input, x.N, tt.want) + } + } +} + +func TestDecodeMalformedNumbers(t *testing.T) { + for _, tt := range []struct { + s string + want string + }{ + {"++99", "expected a digit"}, + {"0..1", "must be followed by one or more digits"}, + {"0.1.2", "Invalid float value"}, + {"1e2.3", "Invalid float value"}, + {"1e2e3", "Invalid float value"}, + {"_123", "expected value"}, + {"123_", "surrounded by digits"}, + {"1._23", "surrounded by digits"}, + {"1e__23", "surrounded by digits"}, + {"123.", "must be followed by one or more digits"}, + {"1.e2", "must be followed by one or more digits"}, + } { + var x struct{ N interface{} } + input := "n = " + tt.s + _, err := Decode(input, &x) + if err == nil { + t.Errorf("Decode(%q): got nil, want error containing %q", + input, tt.want) + continue + } + if !strings.Contains(err.Error(), tt.want) { + t.Errorf("Decode(%q): got %q, want error containing %q", + input, err, tt.want) + } + } +} + +func TestDecodeBadValues(t *testing.T) { + for _, tt := range []struct { + v interface{} + want string + }{ + {3, "non-pointer int"}, + {(*int)(nil), "nil"}, + } { + _, err := Decode(`x = 3`, tt.v) + if err == nil { + t.Errorf("Decode(%v): got nil; want error containing %q", + tt.v, tt.want) + continue + } + if !strings.Contains(err.Error(), tt.want) { + t.Errorf("Decode(%v): got %q; want error containing %q", + tt.v, err, tt.want) + } + } +} + +func TestUnmarshaler(t *testing.T) { + + var tomlBlob = ` +[dishes.hamboogie] +name = "Hamboogie with fries" +price = 10.99 + +[[dishes.hamboogie.ingredients]] +name = "Bread Bun" + +[[dishes.hamboogie.ingredients]] +name = "Lettuce" + +[[dishes.hamboogie.ingredients]] +name = "Real Beef Patty" + +[[dishes.hamboogie.ingredients]] +name = "Tomato" + +[dishes.eggsalad] +name = "Egg Salad with rice" +price = 3.99 + +[[dishes.eggsalad.ingredients]] +name = "Egg" + +[[dishes.eggsalad.ingredients]] +name = "Mayo" + +[[dishes.eggsalad.ingredients]] +name = "Rice" +` + m := &menu{} + if _, err := Decode(tomlBlob, m); err != nil { + t.Fatal(err) + } + + if len(m.Dishes) != 2 { + t.Log("two dishes should be loaded with UnmarshalTOML()") + t.Errorf("expected %d but got %d", 2, len(m.Dishes)) + } + + eggSalad := m.Dishes["eggsalad"] + if _, ok := interface{}(eggSalad).(dish); !ok { + t.Errorf("expected a dish") + } + + if eggSalad.Name != "Egg Salad with rice" { + t.Errorf("expected the dish to be named 'Egg Salad with rice'") + } + + if len(eggSalad.Ingredients) != 3 { + t.Log("dish should be loaded with UnmarshalTOML()") + t.Errorf("expected %d but got %d", 3, len(eggSalad.Ingredients)) + } + + found := false + for _, i := range eggSalad.Ingredients { + if i.Name == "Rice" { + found = true + break + } + } + if !found { + t.Error("Rice was not loaded in UnmarshalTOML()") + } + + // test on a value - must be passed as * + o := menu{} + if _, err := Decode(tomlBlob, &o); err != nil { + t.Fatal(err) + } + +} + +func TestDecodeInlineTable(t *testing.T) { + input := ` +[CookieJar] +Types = {Chocolate = "yummy", Oatmeal = "best ever"} + +[Seasons] +Locations = {NY = {Temp = "not cold", Rating = 4}, MI = {Temp = "freezing", Rating = 9}} +` + type cookieJar struct { + Types map[string]string + } + type properties struct { + Temp string + Rating int + } + type seasons struct { + Locations map[string]properties + } + type wrapper struct { + CookieJar cookieJar + Seasons seasons + } + var got wrapper + + meta, err := Decode(input, &got) + if err != nil { + t.Fatal(err) + } + want := wrapper{ + CookieJar: cookieJar{ + Types: map[string]string{ + "Chocolate": "yummy", + "Oatmeal": "best ever", + }, + }, + Seasons: seasons{ + Locations: map[string]properties{ + "NY": { + Temp: "not cold", + Rating: 4, + }, + "MI": { + Temp: "freezing", + Rating: 9, + }, + }, + }, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("after decode, got:\n\n%#v\n\nwant:\n\n%#v", got, want) + } + if len(meta.keys) != 12 { + t.Errorf("after decode, got %d meta keys; want 12", len(meta.keys)) + } + if len(meta.types) != 12 { + t.Errorf("after decode, got %d meta types; want 12", len(meta.types)) + } +} + +func TestDecodeInlineTableArray(t *testing.T) { + type point struct { + X, Y, Z int + } + var got struct { + Points []point + } + // Example inline table array from the spec. + const in = ` +points = [ { x = 1, y = 2, z = 3 }, + { x = 7, y = 8, z = 9 }, + { x = 2, y = 4, z = 8 } ] + +` + if _, err := Decode(in, &got); err != nil { + t.Fatal(err) + } + want := []point{ + {X: 1, Y: 2, Z: 3}, + {X: 7, Y: 8, Z: 9}, + {X: 2, Y: 4, Z: 8}, + } + if !reflect.DeepEqual(got.Points, want) { + t.Errorf("got %#v; want %#v", got.Points, want) + } +} + +func TestDecodeMalformedInlineTable(t *testing.T) { + for _, tt := range []struct { + s string + want string + }{ + {"{,}", "unexpected comma"}, + {"{x = 3 y = 4}", "expected a comma or an inline table terminator"}, + {"{x=3,,y=4}", "unexpected comma"}, + {"{x=3,\ny=4}", "newlines not allowed"}, + {"{x=3\n,y=4}", "newlines not allowed"}, + } { + var x struct{ A map[string]int } + input := "a = " + tt.s + _, err := Decode(input, &x) + if err == nil { + t.Errorf("Decode(%q): got nil, want error containing %q", + input, tt.want) + continue + } + if !strings.Contains(err.Error(), tt.want) { + t.Errorf("Decode(%q): got %q, want error containing %q", + input, err, tt.want) + } + } +} + +type menu struct { + Dishes map[string]dish +} + +func (m *menu) UnmarshalTOML(p interface{}) error { + m.Dishes = make(map[string]dish) + data, _ := p.(map[string]interface{}) + dishes := data["dishes"].(map[string]interface{}) + for n, v := range dishes { + if d, ok := v.(map[string]interface{}); ok { + nd := dish{} + nd.UnmarshalTOML(d) + m.Dishes[n] = nd + } else { + return fmt.Errorf("not a dish") + } + } + return nil +} + +type dish struct { + Name string + Price float32 + Ingredients []ingredient +} + +func (d *dish) UnmarshalTOML(p interface{}) error { + data, _ := p.(map[string]interface{}) + d.Name, _ = data["name"].(string) + d.Price, _ = data["price"].(float32) + ingredients, _ := data["ingredients"].([]map[string]interface{}) + for _, e := range ingredients { + n, _ := interface{}(e).(map[string]interface{}) + name, _ := n["name"].(string) + i := ingredient{name} + d.Ingredients = append(d.Ingredients, i) + } + return nil +} + +type ingredient struct { + Name string +} + +func TestDecodeSlices(t *testing.T) { + type T struct { + S []string + } + for i, tt := range []struct { + v T + input string + want T + }{ + {T{}, "", T{}}, + {T{[]string{}}, "", T{[]string{}}}, + {T{[]string{"a", "b"}}, "", T{[]string{"a", "b"}}}, + {T{}, "S = []", T{[]string{}}}, + {T{[]string{}}, "S = []", T{[]string{}}}, + {T{[]string{"a", "b"}}, "S = []", T{[]string{}}}, + {T{}, `S = ["x"]`, T{[]string{"x"}}}, + {T{[]string{}}, `S = ["x"]`, T{[]string{"x"}}}, + {T{[]string{"a", "b"}}, `S = ["x"]`, T{[]string{"x"}}}, + } { + if _, err := Decode(tt.input, &tt.v); err != nil { + t.Errorf("[%d] %s", i, err) + continue + } + if !reflect.DeepEqual(tt.v, tt.want) { + t.Errorf("[%d] got %#v; want %#v", i, tt.v, tt.want) + } + } +} + +func TestDecodePrimitive(t *testing.T) { + type S struct { + P Primitive + } + type T struct { + S []int + } + slicep := func(s []int) *[]int { return &s } + arrayp := func(a [2]int) *[2]int { return &a } + mapp := func(m map[string]int) *map[string]int { return &m } + for i, tt := range []struct { + v interface{} + input string + want interface{} + }{ + // slices + {slicep(nil), "", slicep(nil)}, + {slicep([]int{}), "", slicep([]int{})}, + {slicep([]int{1, 2, 3}), "", slicep([]int{1, 2, 3})}, + {slicep(nil), "P = [1,2]", slicep([]int{1, 2})}, + {slicep([]int{}), "P = [1,2]", slicep([]int{1, 2})}, + {slicep([]int{1, 2, 3}), "P = [1,2]", slicep([]int{1, 2})}, + + // arrays + {arrayp([2]int{2, 3}), "", arrayp([2]int{2, 3})}, + {arrayp([2]int{2, 3}), "P = [3,4]", arrayp([2]int{3, 4})}, + + // maps + {mapp(nil), "", mapp(nil)}, + {mapp(map[string]int{}), "", mapp(map[string]int{})}, + {mapp(map[string]int{"a": 1}), "", mapp(map[string]int{"a": 1})}, + {mapp(nil), "[P]\na = 2", mapp(map[string]int{"a": 2})}, + {mapp(map[string]int{}), "[P]\na = 2", mapp(map[string]int{"a": 2})}, + {mapp(map[string]int{"a": 1, "b": 3}), "[P]\na = 2", mapp(map[string]int{"a": 2, "b": 3})}, + + // structs + {&T{nil}, "[P]", &T{nil}}, + {&T{[]int{}}, "[P]", &T{[]int{}}}, + {&T{[]int{1, 2, 3}}, "[P]", &T{[]int{1, 2, 3}}}, + {&T{nil}, "[P]\nS = [1,2]", &T{[]int{1, 2}}}, + {&T{[]int{}}, "[P]\nS = [1,2]", &T{[]int{1, 2}}}, + {&T{[]int{1, 2, 3}}, "[P]\nS = [1,2]", &T{[]int{1, 2}}}, + } { + var s S + md, err := Decode(tt.input, &s) + if err != nil { + t.Errorf("[%d] Decode error: %s", i, err) + continue + } + if err := md.PrimitiveDecode(s.P, tt.v); err != nil { + t.Errorf("[%d] PrimitiveDecode error: %s", i, err) + continue + } + if !reflect.DeepEqual(tt.v, tt.want) { + t.Errorf("[%d] got %#v; want %#v", i, tt.v, tt.want) + } + } +} + +func TestDecodeErrors(t *testing.T) { + for _, s := range []string{ + `x="`, + `x='`, + `x='''`, + + // Cases found by fuzzing in + // https://github.com/BurntSushi/toml/issues/155. + `""�`, // used to panic with index out of range + `e="""`, // used to hang + } { + var x struct{} + _, err := Decode(s, &x) + if err == nil { + t.Errorf("Decode(%q): got nil error", s) + } + } +} + +// Test for https://github.com/BurntSushi/toml/pull/166. +func TestDecodeBoolArray(t *testing.T) { + for _, tt := range []struct { + s string + got interface{} + want interface{} + }{ + { + "a = [true, false]", + &struct{ A []bool }{}, + &struct{ A []bool }{[]bool{true, false}}, + }, + { + "a = {a = true, b = false}", + &struct{ A map[string]bool }{}, + &struct{ A map[string]bool }{map[string]bool{"a": true, "b": false}}, + }, + } { + if _, err := Decode(tt.s, tt.got); err != nil { + t.Errorf("Decode(%q): %s", tt.s, err) + continue + } + if !reflect.DeepEqual(tt.got, tt.want) { + t.Errorf("Decode(%q): got %#v; want %#v", tt.s, tt.got, tt.want) + } + } +} + +func ExampleMetaData_PrimitiveDecode() { + var md MetaData + var err error + + var tomlBlob = ` +ranking = ["Springsteen", "J Geils"] + +[bands.Springsteen] +started = 1973 +albums = ["Greetings", "WIESS", "Born to Run", "Darkness"] + +[bands."J Geils"] +started = 1970 +albums = ["The J. Geils Band", "Full House", "Blow Your Face Out"] +` + + type band struct { + Started int + Albums []string + } + type classics struct { + Ranking []string + Bands map[string]Primitive + } + + // Do the initial decode. Reflection is delayed on Primitive values. + var music classics + if md, err = Decode(tomlBlob, &music); err != nil { + log.Fatal(err) + } + + // MetaData still includes information on Primitive values. + fmt.Printf("Is `bands.Springsteen` defined? %v\n", + md.IsDefined("bands", "Springsteen")) + + // Decode primitive data into Go values. + for _, artist := range music.Ranking { + // A band is a primitive value, so we need to decode it to get a + // real `band` value. + primValue := music.Bands[artist] + + var aBand band + if err = md.PrimitiveDecode(primValue, &aBand); err != nil { + log.Fatal(err) + } + fmt.Printf("%s started in %d.\n", artist, aBand.Started) + } + // Check to see if there were any fields left undecoded. + // Note that this won't be empty before decoding the Primitive value! + fmt.Printf("Undecoded: %q\n", md.Undecoded()) + + // Output: + // Is `bands.Springsteen` defined? true + // Springsteen started in 1973. + // J Geils started in 1970. + // Undecoded: [] +} + +func ExampleDecode() { + var tomlBlob = ` +# Some comments. +[alpha] +ip = "10.0.0.1" + + [alpha.config] + Ports = [ 8001, 8002 ] + Location = "Toronto" + Created = 1987-07-05T05:45:00Z + +[beta] +ip = "10.0.0.2" + + [beta.config] + Ports = [ 9001, 9002 ] + Location = "New Jersey" + Created = 1887-01-05T05:55:00Z +` + + type serverConfig struct { + Ports []int + Location string + Created time.Time + } + + type server struct { + IP string `toml:"ip,omitempty"` + Config serverConfig `toml:"config"` + } + + type servers map[string]server + + var config servers + if _, err := Decode(tomlBlob, &config); err != nil { + log.Fatal(err) + } + + for _, name := range []string{"alpha", "beta"} { + s := config[name] + fmt.Printf("Server: %s (ip: %s) in %s created on %s\n", + name, s.IP, s.Config.Location, + s.Config.Created.Format("2006-01-02")) + fmt.Printf("Ports: %v\n", s.Config.Ports) + } + + // Output: + // Server: alpha (ip: 10.0.0.1) in Toronto created on 1987-07-05 + // Ports: [8001 8002] + // Server: beta (ip: 10.0.0.2) in New Jersey created on 1887-01-05 + // Ports: [9001 9002] +} + +type duration struct { + time.Duration +} + +func (d *duration) UnmarshalText(text []byte) error { + var err error + d.Duration, err = time.ParseDuration(string(text)) + return err +} + +// Example Unmarshaler shows how to decode TOML strings into your own +// custom data type. +func Example_unmarshaler() { + blob := ` +[[song]] +name = "Thunder Road" +duration = "4m49s" + +[[song]] +name = "Stairway to Heaven" +duration = "8m03s" +` + type song struct { + Name string + Duration duration + } + type songs struct { + Song []song + } + var favorites songs + if _, err := Decode(blob, &favorites); err != nil { + log.Fatal(err) + } + + // Code to implement the TextUnmarshaler interface for `duration`: + // + // type duration struct { + // time.Duration + // } + // + // func (d *duration) UnmarshalText(text []byte) error { + // var err error + // d.Duration, err = time.ParseDuration(string(text)) + // return err + // } + + for _, s := range favorites.Song { + fmt.Printf("%s (%s)\n", s.Name, s.Duration) + } + // Output: + // Thunder Road (4m49s) + // Stairway to Heaven (8m3s) +} + +// Example StrictDecoding shows how to detect whether there are keys in the +// TOML document that weren't decoded into the value given. This is useful +// for returning an error to the user if they've included extraneous fields +// in their configuration. +func Example_strictDecoding() { + var blob = ` +key1 = "value1" +key2 = "value2" +key3 = "value3" +` + type config struct { + Key1 string + Key3 string + } + + var conf config + md, err := Decode(blob, &conf) + if err != nil { + log.Fatal(err) + } + fmt.Printf("Undecoded keys: %q\n", md.Undecoded()) + // Output: + // Undecoded keys: ["key2"] +} + +// Example UnmarshalTOML shows how to implement a struct type that knows how to +// unmarshal itself. The struct must take full responsibility for mapping the +// values passed into the struct. The method may be used with interfaces in a +// struct in cases where the actual type is not known until the data is +// examined. +func Example_unmarshalTOML() { + + var blob = ` +[[parts]] +type = "valve" +id = "valve-1" +size = 1.2 +rating = 4 + +[[parts]] +type = "valve" +id = "valve-2" +size = 2.1 +rating = 5 + +[[parts]] +type = "pipe" +id = "pipe-1" +length = 2.1 +diameter = 12 + +[[parts]] +type = "cable" +id = "cable-1" +length = 12 +rating = 3.1 +` + o := &order{} + err := Unmarshal([]byte(blob), o) + if err != nil { + log.Fatal(err) + } + + fmt.Println(len(o.parts)) + + for _, part := range o.parts { + fmt.Println(part.Name()) + } + + // Code to implement UmarshalJSON. + + // type order struct { + // // NOTE `order.parts` is a private slice of type `part` which is an + // // interface and may only be loaded from toml using the + // // UnmarshalTOML() method of the Umarshaler interface. + // parts parts + // } + + // func (o *order) UnmarshalTOML(data interface{}) error { + + // // NOTE the example below contains detailed type casting to show how + // // the 'data' is retrieved. In operational use, a type cast wrapper + // // may be preferred e.g. + // // + // // func AsMap(v interface{}) (map[string]interface{}, error) { + // // return v.(map[string]interface{}) + // // } + // // + // // resulting in: + // // d, _ := AsMap(data) + // // + + // d, _ := data.(map[string]interface{}) + // parts, _ := d["parts"].([]map[string]interface{}) + + // for _, p := range parts { + + // typ, _ := p["type"].(string) + // id, _ := p["id"].(string) + + // // detect the type of part and handle each case + // switch p["type"] { + // case "valve": + + // size := float32(p["size"].(float64)) + // rating := int(p["rating"].(int64)) + + // valve := &valve{ + // Type: typ, + // ID: id, + // Size: size, + // Rating: rating, + // } + + // o.parts = append(o.parts, valve) + + // case "pipe": + + // length := float32(p["length"].(float64)) + // diameter := int(p["diameter"].(int64)) + + // pipe := &pipe{ + // Type: typ, + // ID: id, + // Length: length, + // Diameter: diameter, + // } + + // o.parts = append(o.parts, pipe) + + // case "cable": + + // length := int(p["length"].(int64)) + // rating := float32(p["rating"].(float64)) + + // cable := &cable{ + // Type: typ, + // ID: id, + // Length: length, + // Rating: rating, + // } + + // o.parts = append(o.parts, cable) + + // } + // } + + // return nil + // } + + // type parts []part + + // type part interface { + // Name() string + // } + + // type valve struct { + // Type string + // ID string + // Size float32 + // Rating int + // } + + // func (v *valve) Name() string { + // return fmt.Sprintf("VALVE: %s", v.ID) + // } + + // type pipe struct { + // Type string + // ID string + // Length float32 + // Diameter int + // } + + // func (p *pipe) Name() string { + // return fmt.Sprintf("PIPE: %s", p.ID) + // } + + // type cable struct { + // Type string + // ID string + // Length int + // Rating float32 + // } + + // func (c *cable) Name() string { + // return fmt.Sprintf("CABLE: %s", c.ID) + // } + + // Output: + // 4 + // VALVE: valve-1 + // VALVE: valve-2 + // PIPE: pipe-1 + // CABLE: cable-1 + +} + +type order struct { + // NOTE `order.parts` is a private slice of type `part` which is an + // interface and may only be loaded from toml using the UnmarshalTOML() + // method of the Umarshaler interface. + parts parts +} + +func (o *order) UnmarshalTOML(data interface{}) error { + + // NOTE the example below contains detailed type casting to show how + // the 'data' is retrieved. In operational use, a type cast wrapper + // may be preferred e.g. + // + // func AsMap(v interface{}) (map[string]interface{}, error) { + // return v.(map[string]interface{}) + // } + // + // resulting in: + // d, _ := AsMap(data) + // + + d, _ := data.(map[string]interface{}) + parts, _ := d["parts"].([]map[string]interface{}) + + for _, p := range parts { + + typ, _ := p["type"].(string) + id, _ := p["id"].(string) + + // detect the type of part and handle each case + switch p["type"] { + case "valve": + + size := float32(p["size"].(float64)) + rating := int(p["rating"].(int64)) + + valve := &valve{ + Type: typ, + ID: id, + Size: size, + Rating: rating, + } + + o.parts = append(o.parts, valve) + + case "pipe": + + length := float32(p["length"].(float64)) + diameter := int(p["diameter"].(int64)) + + pipe := &pipe{ + Type: typ, + ID: id, + Length: length, + Diameter: diameter, + } + + o.parts = append(o.parts, pipe) + + case "cable": + + length := int(p["length"].(int64)) + rating := float32(p["rating"].(float64)) + + cable := &cable{ + Type: typ, + ID: id, + Length: length, + Rating: rating, + } + + o.parts = append(o.parts, cable) + + } + } + + return nil +} + +type parts []part + +type part interface { + Name() string +} + +type valve struct { + Type string + ID string + Size float32 + Rating int +} + +func (v *valve) Name() string { + return fmt.Sprintf("VALVE: %s", v.ID) +} + +type pipe struct { + Type string + ID string + Length float32 + Diameter int +} + +func (p *pipe) Name() string { + return fmt.Sprintf("PIPE: %s", p.ID) +} + +type cable struct { + Type string + ID string + Length int + Rating float32 +} + +func (c *cable) Name() string { + return fmt.Sprintf("CABLE: %s", c.ID) +} diff --git a/vendor/github.com/BurntSushi/toml/doc.go b/vendor/github.com/BurntSushi/toml/doc.go new file mode 100644 index 0000000..b371f39 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/doc.go @@ -0,0 +1,27 @@ +/* +Package toml provides facilities for decoding and encoding TOML configuration +files via reflection. There is also support for delaying decoding with +the Primitive type, and querying the set of keys in a TOML document with the +MetaData type. + +The specification implemented: https://github.com/toml-lang/toml + +The sub-command github.com/BurntSushi/toml/cmd/tomlv can be used to verify +whether a file is a valid TOML document. It can also be used to print the +type of each key in a TOML document. + +Testing + +There are two important types of tests used for this package. The first is +contained inside '*_test.go' files and uses the standard Go unit testing +framework. These tests are primarily devoted to holistically testing the +decoder and encoder. + +The second type of testing is used to verify the implementation's adherence +to the TOML specification. These tests have been factored into their own +project: https://github.com/BurntSushi/toml-test + +The reason the tests are in a separate project is so that they can be used by +any implementation of TOML. Namely, it is language agnostic. +*/ +package toml diff --git a/vendor/github.com/BurntSushi/toml/encode.go b/vendor/github.com/BurntSushi/toml/encode.go new file mode 100644 index 0000000..d905c21 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/encode.go @@ -0,0 +1,568 @@ +package toml + +import ( + "bufio" + "errors" + "fmt" + "io" + "reflect" + "sort" + "strconv" + "strings" + "time" +) + +type tomlEncodeError struct{ error } + +var ( + errArrayMixedElementTypes = errors.New( + "toml: cannot encode array with mixed element types") + errArrayNilElement = errors.New( + "toml: cannot encode array with nil element") + errNonString = errors.New( + "toml: cannot encode a map with non-string key type") + errAnonNonStruct = errors.New( + "toml: cannot encode an anonymous field that is not a struct") + errArrayNoTable = errors.New( + "toml: TOML array element cannot contain a table") + errNoKey = errors.New( + "toml: top-level values must be Go maps or structs") + errAnything = errors.New("") // used in testing +) + +var quotedReplacer = strings.NewReplacer( + "\t", "\\t", + "\n", "\\n", + "\r", "\\r", + "\"", "\\\"", + "\\", "\\\\", +) + +// Encoder controls the encoding of Go values to a TOML document to some +// io.Writer. +// +// The indentation level can be controlled with the Indent field. +type Encoder struct { + // A single indentation level. By default it is two spaces. + Indent string + + // hasWritten is whether we have written any output to w yet. + hasWritten bool + w *bufio.Writer +} + +// NewEncoder returns a TOML encoder that encodes Go values to the io.Writer +// given. By default, a single indentation level is 2 spaces. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ + w: bufio.NewWriter(w), + Indent: " ", + } +} + +// Encode writes a TOML representation of the Go value to the underlying +// io.Writer. If the value given cannot be encoded to a valid TOML document, +// then an error is returned. +// +// The mapping between Go values and TOML values should be precisely the same +// as for the Decode* functions. Similarly, the TextMarshaler interface is +// supported by encoding the resulting bytes as strings. (If you want to write +// arbitrary binary data then you will need to use something like base64 since +// TOML does not have any binary types.) +// +// When encoding TOML hashes (i.e., Go maps or structs), keys without any +// sub-hashes are encoded first. +// +// If a Go map is encoded, then its keys are sorted alphabetically for +// deterministic output. More control over this behavior may be provided if +// there is demand for it. +// +// Encoding Go values without a corresponding TOML representation---like map +// types with non-string keys---will cause an error to be returned. Similarly +// for mixed arrays/slices, arrays/slices with nil elements, embedded +// non-struct types and nested slices containing maps or structs. +// (e.g., [][]map[string]string is not allowed but []map[string]string is OK +// and so is []map[string][]string.) +func (enc *Encoder) Encode(v interface{}) error { + rv := eindirect(reflect.ValueOf(v)) + if err := enc.safeEncode(Key([]string{}), rv); err != nil { + return err + } + return enc.w.Flush() +} + +func (enc *Encoder) safeEncode(key Key, rv reflect.Value) (err error) { + defer func() { + if r := recover(); r != nil { + if terr, ok := r.(tomlEncodeError); ok { + err = terr.error + return + } + panic(r) + } + }() + enc.encode(key, rv) + return nil +} + +func (enc *Encoder) encode(key Key, rv reflect.Value) { + // Special case. Time needs to be in ISO8601 format. + // Special case. If we can marshal the type to text, then we used that. + // Basically, this prevents the encoder for handling these types as + // generic structs (or whatever the underlying type of a TextMarshaler is). + switch rv.Interface().(type) { + case time.Time, TextMarshaler: + enc.keyEqElement(key, rv) + return + } + + k := rv.Kind() + switch k { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, + reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, + reflect.Uint64, + reflect.Float32, reflect.Float64, reflect.String, reflect.Bool: + enc.keyEqElement(key, rv) + case reflect.Array, reflect.Slice: + if typeEqual(tomlArrayHash, tomlTypeOfGo(rv)) { + enc.eArrayOfTables(key, rv) + } else { + enc.keyEqElement(key, rv) + } + case reflect.Interface: + if rv.IsNil() { + return + } + enc.encode(key, rv.Elem()) + case reflect.Map: + if rv.IsNil() { + return + } + enc.eTable(key, rv) + case reflect.Ptr: + if rv.IsNil() { + return + } + enc.encode(key, rv.Elem()) + case reflect.Struct: + enc.eTable(key, rv) + default: + panic(e("unsupported type for key '%s': %s", key, k)) + } +} + +// eElement encodes any value that can be an array element (primitives and +// arrays). +func (enc *Encoder) eElement(rv reflect.Value) { + switch v := rv.Interface().(type) { + case time.Time: + // Special case time.Time as a primitive. Has to come before + // TextMarshaler below because time.Time implements + // encoding.TextMarshaler, but we need to always use UTC. + enc.wf(v.UTC().Format("2006-01-02T15:04:05Z")) + return + case TextMarshaler: + // Special case. Use text marshaler if it's available for this value. + if s, err := v.MarshalText(); err != nil { + encPanic(err) + } else { + enc.writeQuoted(string(s)) + } + return + } + switch rv.Kind() { + case reflect.Bool: + enc.wf(strconv.FormatBool(rv.Bool())) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, + reflect.Int64: + enc.wf(strconv.FormatInt(rv.Int(), 10)) + case reflect.Uint, reflect.Uint8, reflect.Uint16, + reflect.Uint32, reflect.Uint64: + enc.wf(strconv.FormatUint(rv.Uint(), 10)) + case reflect.Float32: + enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 32))) + case reflect.Float64: + enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 64))) + case reflect.Array, reflect.Slice: + enc.eArrayOrSliceElement(rv) + case reflect.Interface: + enc.eElement(rv.Elem()) + case reflect.String: + enc.writeQuoted(rv.String()) + default: + panic(e("unexpected primitive type: %s", rv.Kind())) + } +} + +// By the TOML spec, all floats must have a decimal with at least one +// number on either side. +func floatAddDecimal(fstr string) string { + if !strings.Contains(fstr, ".") { + return fstr + ".0" + } + return fstr +} + +func (enc *Encoder) writeQuoted(s string) { + enc.wf("\"%s\"", quotedReplacer.Replace(s)) +} + +func (enc *Encoder) eArrayOrSliceElement(rv reflect.Value) { + length := rv.Len() + enc.wf("[") + for i := 0; i < length; i++ { + elem := rv.Index(i) + enc.eElement(elem) + if i != length-1 { + enc.wf(", ") + } + } + enc.wf("]") +} + +func (enc *Encoder) eArrayOfTables(key Key, rv reflect.Value) { + if len(key) == 0 { + encPanic(errNoKey) + } + for i := 0; i < rv.Len(); i++ { + trv := rv.Index(i) + if isNil(trv) { + continue + } + panicIfInvalidKey(key) + enc.newline() + enc.wf("%s[[%s]]", enc.indentStr(key), key.maybeQuotedAll()) + enc.newline() + enc.eMapOrStruct(key, trv) + } +} + +func (enc *Encoder) eTable(key Key, rv reflect.Value) { + panicIfInvalidKey(key) + if len(key) == 1 { + // Output an extra newline between top-level tables. + // (The newline isn't written if nothing else has been written though.) + enc.newline() + } + if len(key) > 0 { + enc.wf("%s[%s]", enc.indentStr(key), key.maybeQuotedAll()) + enc.newline() + } + enc.eMapOrStruct(key, rv) +} + +func (enc *Encoder) eMapOrStruct(key Key, rv reflect.Value) { + switch rv := eindirect(rv); rv.Kind() { + case reflect.Map: + enc.eMap(key, rv) + case reflect.Struct: + enc.eStruct(key, rv) + default: + panic("eTable: unhandled reflect.Value Kind: " + rv.Kind().String()) + } +} + +func (enc *Encoder) eMap(key Key, rv reflect.Value) { + rt := rv.Type() + if rt.Key().Kind() != reflect.String { + encPanic(errNonString) + } + + // Sort keys so that we have deterministic output. And write keys directly + // underneath this key first, before writing sub-structs or sub-maps. + var mapKeysDirect, mapKeysSub []string + for _, mapKey := range rv.MapKeys() { + k := mapKey.String() + if typeIsHash(tomlTypeOfGo(rv.MapIndex(mapKey))) { + mapKeysSub = append(mapKeysSub, k) + } else { + mapKeysDirect = append(mapKeysDirect, k) + } + } + + var writeMapKeys = func(mapKeys []string) { + sort.Strings(mapKeys) + for _, mapKey := range mapKeys { + mrv := rv.MapIndex(reflect.ValueOf(mapKey)) + if isNil(mrv) { + // Don't write anything for nil fields. + continue + } + enc.encode(key.add(mapKey), mrv) + } + } + writeMapKeys(mapKeysDirect) + writeMapKeys(mapKeysSub) +} + +func (enc *Encoder) eStruct(key Key, rv reflect.Value) { + // Write keys for fields directly under this key first, because if we write + // a field that creates a new table, then all keys under it will be in that + // table (not the one we're writing here). + rt := rv.Type() + var fieldsDirect, fieldsSub [][]int + var addFields func(rt reflect.Type, rv reflect.Value, start []int) + addFields = func(rt reflect.Type, rv reflect.Value, start []int) { + for i := 0; i < rt.NumField(); i++ { + f := rt.Field(i) + // skip unexported fields + if f.PkgPath != "" && !f.Anonymous { + continue + } + frv := rv.Field(i) + if f.Anonymous { + t := f.Type + switch t.Kind() { + case reflect.Struct: + // Treat anonymous struct fields with + // tag names as though they are not + // anonymous, like encoding/json does. + if getOptions(f.Tag).name == "" { + addFields(t, frv, f.Index) + continue + } + case reflect.Ptr: + if t.Elem().Kind() == reflect.Struct && + getOptions(f.Tag).name == "" { + if !frv.IsNil() { + addFields(t.Elem(), frv.Elem(), f.Index) + } + continue + } + // Fall through to the normal field encoding logic below + // for non-struct anonymous fields. + } + } + + if typeIsHash(tomlTypeOfGo(frv)) { + fieldsSub = append(fieldsSub, append(start, f.Index...)) + } else { + fieldsDirect = append(fieldsDirect, append(start, f.Index...)) + } + } + } + addFields(rt, rv, nil) + + var writeFields = func(fields [][]int) { + for _, fieldIndex := range fields { + sft := rt.FieldByIndex(fieldIndex) + sf := rv.FieldByIndex(fieldIndex) + if isNil(sf) { + // Don't write anything for nil fields. + continue + } + + opts := getOptions(sft.Tag) + if opts.skip { + continue + } + keyName := sft.Name + if opts.name != "" { + keyName = opts.name + } + if opts.omitempty && isEmpty(sf) { + continue + } + if opts.omitzero && isZero(sf) { + continue + } + + enc.encode(key.add(keyName), sf) + } + } + writeFields(fieldsDirect) + writeFields(fieldsSub) +} + +// tomlTypeName returns the TOML type name of the Go value's type. It is +// used to determine whether the types of array elements are mixed (which is +// forbidden). If the Go value is nil, then it is illegal for it to be an array +// element, and valueIsNil is returned as true. + +// Returns the TOML type of a Go value. The type may be `nil`, which means +// no concrete TOML type could be found. +func tomlTypeOfGo(rv reflect.Value) tomlType { + if isNil(rv) || !rv.IsValid() { + return nil + } + switch rv.Kind() { + case reflect.Bool: + return tomlBool + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, + reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, + reflect.Uint64: + return tomlInteger + case reflect.Float32, reflect.Float64: + return tomlFloat + case reflect.Array, reflect.Slice: + if typeEqual(tomlHash, tomlArrayType(rv)) { + return tomlArrayHash + } + return tomlArray + case reflect.Ptr, reflect.Interface: + return tomlTypeOfGo(rv.Elem()) + case reflect.String: + return tomlString + case reflect.Map: + return tomlHash + case reflect.Struct: + switch rv.Interface().(type) { + case time.Time: + return tomlDatetime + case TextMarshaler: + return tomlString + default: + return tomlHash + } + default: + panic("unexpected reflect.Kind: " + rv.Kind().String()) + } +} + +// tomlArrayType returns the element type of a TOML array. The type returned +// may be nil if it cannot be determined (e.g., a nil slice or a zero length +// slize). This function may also panic if it finds a type that cannot be +// expressed in TOML (such as nil elements, heterogeneous arrays or directly +// nested arrays of tables). +func tomlArrayType(rv reflect.Value) tomlType { + if isNil(rv) || !rv.IsValid() || rv.Len() == 0 { + return nil + } + firstType := tomlTypeOfGo(rv.Index(0)) + if firstType == nil { + encPanic(errArrayNilElement) + } + + rvlen := rv.Len() + for i := 1; i < rvlen; i++ { + elem := rv.Index(i) + switch elemType := tomlTypeOfGo(elem); { + case elemType == nil: + encPanic(errArrayNilElement) + case !typeEqual(firstType, elemType): + encPanic(errArrayMixedElementTypes) + } + } + // If we have a nested array, then we must make sure that the nested + // array contains ONLY primitives. + // This checks arbitrarily nested arrays. + if typeEqual(firstType, tomlArray) || typeEqual(firstType, tomlArrayHash) { + nest := tomlArrayType(eindirect(rv.Index(0))) + if typeEqual(nest, tomlHash) || typeEqual(nest, tomlArrayHash) { + encPanic(errArrayNoTable) + } + } + return firstType +} + +type tagOptions struct { + skip bool // "-" + name string + omitempty bool + omitzero bool +} + +func getOptions(tag reflect.StructTag) tagOptions { + t := tag.Get("toml") + if t == "-" { + return tagOptions{skip: true} + } + var opts tagOptions + parts := strings.Split(t, ",") + opts.name = parts[0] + for _, s := range parts[1:] { + switch s { + case "omitempty": + opts.omitempty = true + case "omitzero": + opts.omitzero = true + } + } + return opts +} + +func isZero(rv reflect.Value) bool { + switch rv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return rv.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return rv.Uint() == 0 + case reflect.Float32, reflect.Float64: + return rv.Float() == 0.0 + } + return false +} + +func isEmpty(rv reflect.Value) bool { + switch rv.Kind() { + case reflect.Array, reflect.Slice, reflect.Map, reflect.String: + return rv.Len() == 0 + case reflect.Bool: + return !rv.Bool() + } + return false +} + +func (enc *Encoder) newline() { + if enc.hasWritten { + enc.wf("\n") + } +} + +func (enc *Encoder) keyEqElement(key Key, val reflect.Value) { + if len(key) == 0 { + encPanic(errNoKey) + } + panicIfInvalidKey(key) + enc.wf("%s%s = ", enc.indentStr(key), key.maybeQuoted(len(key)-1)) + enc.eElement(val) + enc.newline() +} + +func (enc *Encoder) wf(format string, v ...interface{}) { + if _, err := fmt.Fprintf(enc.w, format, v...); err != nil { + encPanic(err) + } + enc.hasWritten = true +} + +func (enc *Encoder) indentStr(key Key) string { + return strings.Repeat(enc.Indent, len(key)-1) +} + +func encPanic(err error) { + panic(tomlEncodeError{err}) +} + +func eindirect(v reflect.Value) reflect.Value { + switch v.Kind() { + case reflect.Ptr, reflect.Interface: + return eindirect(v.Elem()) + default: + return v + } +} + +func isNil(rv reflect.Value) bool { + switch rv.Kind() { + case reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return rv.IsNil() + default: + return false + } +} + +func panicIfInvalidKey(key Key) { + for _, k := range key { + if len(k) == 0 { + encPanic(e("Key '%s' is not a valid table name. Key names "+ + "cannot be empty.", key.maybeQuotedAll())) + } + } +} + +func isValidKeyName(s string) bool { + return len(s) != 0 +} diff --git a/vendor/github.com/BurntSushi/toml/encode_test.go b/vendor/github.com/BurntSushi/toml/encode_test.go new file mode 100644 index 0000000..673b7b0 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/encode_test.go @@ -0,0 +1,615 @@ +package toml + +import ( + "bytes" + "fmt" + "log" + "net" + "testing" + "time" +) + +func TestEncodeRoundTrip(t *testing.T) { + type Config struct { + Age int + Cats []string + Pi float64 + Perfection []int + DOB time.Time + Ipaddress net.IP + } + + var inputs = Config{ + 13, + []string{"one", "two", "three"}, + 3.145, + []int{11, 2, 3, 4}, + time.Now(), + net.ParseIP("192.168.59.254"), + } + + var firstBuffer bytes.Buffer + e := NewEncoder(&firstBuffer) + err := e.Encode(inputs) + if err != nil { + t.Fatal(err) + } + var outputs Config + if _, err := Decode(firstBuffer.String(), &outputs); err != nil { + t.Logf("Could not decode:\n-----\n%s\n-----\n", + firstBuffer.String()) + t.Fatal(err) + } + + // could test each value individually, but I'm lazy + var secondBuffer bytes.Buffer + e2 := NewEncoder(&secondBuffer) + err = e2.Encode(outputs) + if err != nil { + t.Fatal(err) + } + if firstBuffer.String() != secondBuffer.String() { + t.Error( + firstBuffer.String(), + "\n\n is not identical to\n\n", + secondBuffer.String()) + } +} + +// XXX(burntsushi) +// I think these tests probably should be removed. They are good, but they +// ought to be obsolete by toml-test. +func TestEncode(t *testing.T) { + type Embedded struct { + Int int `toml:"_int"` + } + type NonStruct int + + date := time.Date(2014, 5, 11, 20, 30, 40, 0, time.FixedZone("IST", 3600)) + dateStr := "2014-05-11T19:30:40Z" + + tests := map[string]struct { + input interface{} + wantOutput string + wantError error + }{ + "bool field": { + input: struct { + BoolTrue bool + BoolFalse bool + }{true, false}, + wantOutput: "BoolTrue = true\nBoolFalse = false\n", + }, + "int fields": { + input: struct { + Int int + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + }{1, 2, 3, 4, 5}, + wantOutput: "Int = 1\nInt8 = 2\nInt16 = 3\nInt32 = 4\nInt64 = 5\n", + }, + "uint fields": { + input: struct { + Uint uint + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + }{1, 2, 3, 4, 5}, + wantOutput: "Uint = 1\nUint8 = 2\nUint16 = 3\nUint32 = 4" + + "\nUint64 = 5\n", + }, + "float fields": { + input: struct { + Float32 float32 + Float64 float64 + }{1.5, 2.5}, + wantOutput: "Float32 = 1.5\nFloat64 = 2.5\n", + }, + "string field": { + input: struct{ String string }{"foo"}, + wantOutput: "String = \"foo\"\n", + }, + "string field and unexported field": { + input: struct { + String string + unexported int + }{"foo", 0}, + wantOutput: "String = \"foo\"\n", + }, + "datetime field in UTC": { + input: struct{ Date time.Time }{date}, + wantOutput: fmt.Sprintf("Date = %s\n", dateStr), + }, + "datetime field as primitive": { + // Using a map here to fail if isStructOrMap() returns true for + // time.Time. + input: map[string]interface{}{ + "Date": date, + "Int": 1, + }, + wantOutput: fmt.Sprintf("Date = %s\nInt = 1\n", dateStr), + }, + "array fields": { + input: struct { + IntArray0 [0]int + IntArray3 [3]int + }{[0]int{}, [3]int{1, 2, 3}}, + wantOutput: "IntArray0 = []\nIntArray3 = [1, 2, 3]\n", + }, + "slice fields": { + input: struct{ IntSliceNil, IntSlice0, IntSlice3 []int }{ + nil, []int{}, []int{1, 2, 3}, + }, + wantOutput: "IntSlice0 = []\nIntSlice3 = [1, 2, 3]\n", + }, + "datetime slices": { + input: struct{ DatetimeSlice []time.Time }{ + []time.Time{date, date}, + }, + wantOutput: fmt.Sprintf("DatetimeSlice = [%s, %s]\n", + dateStr, dateStr), + }, + "nested arrays and slices": { + input: struct { + SliceOfArrays [][2]int + ArrayOfSlices [2][]int + SliceOfArraysOfSlices [][2][]int + ArrayOfSlicesOfArrays [2][][2]int + SliceOfMixedArrays [][2]interface{} + ArrayOfMixedSlices [2][]interface{} + }{ + [][2]int{{1, 2}, {3, 4}}, + [2][]int{{1, 2}, {3, 4}}, + [][2][]int{ + { + {1, 2}, {3, 4}, + }, + { + {5, 6}, {7, 8}, + }, + }, + [2][][2]int{ + { + {1, 2}, {3, 4}, + }, + { + {5, 6}, {7, 8}, + }, + }, + [][2]interface{}{ + {1, 2}, {"a", "b"}, + }, + [2][]interface{}{ + {1, 2}, {"a", "b"}, + }, + }, + wantOutput: `SliceOfArrays = [[1, 2], [3, 4]] +ArrayOfSlices = [[1, 2], [3, 4]] +SliceOfArraysOfSlices = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]] +ArrayOfSlicesOfArrays = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]] +SliceOfMixedArrays = [[1, 2], ["a", "b"]] +ArrayOfMixedSlices = [[1, 2], ["a", "b"]] +`, + }, + "empty slice": { + input: struct{ Empty []interface{} }{[]interface{}{}}, + wantOutput: "Empty = []\n", + }, + "(error) slice with element type mismatch (string and integer)": { + input: struct{ Mixed []interface{} }{[]interface{}{1, "a"}}, + wantError: errArrayMixedElementTypes, + }, + "(error) slice with element type mismatch (integer and float)": { + input: struct{ Mixed []interface{} }{[]interface{}{1, 2.5}}, + wantError: errArrayMixedElementTypes, + }, + "slice with elems of differing Go types, same TOML types": { + input: struct { + MixedInts []interface{} + MixedFloats []interface{} + }{ + []interface{}{ + int(1), int8(2), int16(3), int32(4), int64(5), + uint(1), uint8(2), uint16(3), uint32(4), uint64(5), + }, + []interface{}{float32(1.5), float64(2.5)}, + }, + wantOutput: "MixedInts = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5]\n" + + "MixedFloats = [1.5, 2.5]\n", + }, + "(error) slice w/ element type mismatch (one is nested array)": { + input: struct{ Mixed []interface{} }{ + []interface{}{1, []interface{}{2}}, + }, + wantError: errArrayMixedElementTypes, + }, + "(error) slice with 1 nil element": { + input: struct{ NilElement1 []interface{} }{[]interface{}{nil}}, + wantError: errArrayNilElement, + }, + "(error) slice with 1 nil element (and other non-nil elements)": { + input: struct{ NilElement []interface{} }{ + []interface{}{1, nil}, + }, + wantError: errArrayNilElement, + }, + "simple map": { + input: map[string]int{"a": 1, "b": 2}, + wantOutput: "a = 1\nb = 2\n", + }, + "map with interface{} value type": { + input: map[string]interface{}{"a": 1, "b": "c"}, + wantOutput: "a = 1\nb = \"c\"\n", + }, + "map with interface{} value type, some of which are structs": { + input: map[string]interface{}{ + "a": struct{ Int int }{2}, + "b": 1, + }, + wantOutput: "b = 1\n\n[a]\n Int = 2\n", + }, + "nested map": { + input: map[string]map[string]int{ + "a": {"b": 1}, + "c": {"d": 2}, + }, + wantOutput: "[a]\n b = 1\n\n[c]\n d = 2\n", + }, + "nested struct": { + input: struct{ Struct struct{ Int int } }{ + struct{ Int int }{1}, + }, + wantOutput: "[Struct]\n Int = 1\n", + }, + "nested struct and non-struct field": { + input: struct { + Struct struct{ Int int } + Bool bool + }{struct{ Int int }{1}, true}, + wantOutput: "Bool = true\n\n[Struct]\n Int = 1\n", + }, + "2 nested structs": { + input: struct{ Struct1, Struct2 struct{ Int int } }{ + struct{ Int int }{1}, struct{ Int int }{2}, + }, + wantOutput: "[Struct1]\n Int = 1\n\n[Struct2]\n Int = 2\n", + }, + "deeply nested structs": { + input: struct { + Struct1, Struct2 struct{ Struct3 *struct{ Int int } } + }{ + struct{ Struct3 *struct{ Int int } }{&struct{ Int int }{1}}, + struct{ Struct3 *struct{ Int int } }{nil}, + }, + wantOutput: "[Struct1]\n [Struct1.Struct3]\n Int = 1" + + "\n\n[Struct2]\n", + }, + "nested struct with nil struct elem": { + input: struct { + Struct struct{ Inner *struct{ Int int } } + }{ + struct{ Inner *struct{ Int int } }{nil}, + }, + wantOutput: "[Struct]\n", + }, + "nested struct with no fields": { + input: struct { + Struct struct{ Inner struct{} } + }{ + struct{ Inner struct{} }{struct{}{}}, + }, + wantOutput: "[Struct]\n [Struct.Inner]\n", + }, + "struct with tags": { + input: struct { + Struct struct { + Int int `toml:"_int"` + } `toml:"_struct"` + Bool bool `toml:"_bool"` + }{ + struct { + Int int `toml:"_int"` + }{1}, true, + }, + wantOutput: "_bool = true\n\n[_struct]\n _int = 1\n", + }, + "embedded struct": { + input: struct{ Embedded }{Embedded{1}}, + wantOutput: "_int = 1\n", + }, + "embedded *struct": { + input: struct{ *Embedded }{&Embedded{1}}, + wantOutput: "_int = 1\n", + }, + "nested embedded struct": { + input: struct { + Struct struct{ Embedded } `toml:"_struct"` + }{struct{ Embedded }{Embedded{1}}}, + wantOutput: "[_struct]\n _int = 1\n", + }, + "nested embedded *struct": { + input: struct { + Struct struct{ *Embedded } `toml:"_struct"` + }{struct{ *Embedded }{&Embedded{1}}}, + wantOutput: "[_struct]\n _int = 1\n", + }, + "embedded non-struct": { + input: struct{ NonStruct }{5}, + wantOutput: "NonStruct = 5\n", + }, + "array of tables": { + input: struct { + Structs []*struct{ Int int } `toml:"struct"` + }{ + []*struct{ Int int }{{1}, {3}}, + }, + wantOutput: "[[struct]]\n Int = 1\n\n[[struct]]\n Int = 3\n", + }, + "array of tables order": { + input: map[string]interface{}{ + "map": map[string]interface{}{ + "zero": 5, + "arr": []map[string]int{ + { + "friend": 5, + }, + }, + }, + }, + wantOutput: "[map]\n zero = 5\n\n [[map.arr]]\n friend = 5\n", + }, + "(error) top-level slice": { + input: []struct{ Int int }{{1}, {2}, {3}}, + wantError: errNoKey, + }, + "(error) slice of slice": { + input: struct { + Slices [][]struct{ Int int } + }{ + [][]struct{ Int int }{{{1}}, {{2}}, {{3}}}, + }, + wantError: errArrayNoTable, + }, + "(error) map no string key": { + input: map[int]string{1: ""}, + wantError: errNonString, + }, + "(error) empty key name": { + input: map[string]int{"": 1}, + wantError: errAnything, + }, + "(error) empty map name": { + input: map[string]interface{}{ + "": map[string]int{"v": 1}, + }, + wantError: errAnything, + }, + } + for label, test := range tests { + encodeExpected(t, label, test.input, test.wantOutput, test.wantError) + } +} + +func TestEncodeNestedTableArrays(t *testing.T) { + type song struct { + Name string `toml:"name"` + } + type album struct { + Name string `toml:"name"` + Songs []song `toml:"songs"` + } + type springsteen struct { + Albums []album `toml:"albums"` + } + value := springsteen{ + []album{ + {"Born to Run", + []song{{"Jungleland"}, {"Meeting Across the River"}}}, + {"Born in the USA", + []song{{"Glory Days"}, {"Dancing in the Dark"}}}, + }, + } + expected := `[[albums]] + name = "Born to Run" + + [[albums.songs]] + name = "Jungleland" + + [[albums.songs]] + name = "Meeting Across the River" + +[[albums]] + name = "Born in the USA" + + [[albums.songs]] + name = "Glory Days" + + [[albums.songs]] + name = "Dancing in the Dark" +` + encodeExpected(t, "nested table arrays", value, expected, nil) +} + +func TestEncodeArrayHashWithNormalHashOrder(t *testing.T) { + type Alpha struct { + V int + } + type Beta struct { + V int + } + type Conf struct { + V int + A Alpha + B []Beta + } + + val := Conf{ + V: 1, + A: Alpha{2}, + B: []Beta{{3}}, + } + expected := "V = 1\n\n[A]\n V = 2\n\n[[B]]\n V = 3\n" + encodeExpected(t, "array hash with normal hash order", val, expected, nil) +} + +func TestEncodeWithOmitEmpty(t *testing.T) { + type simple struct { + Bool bool `toml:"bool,omitempty"` + String string `toml:"string,omitempty"` + Array [0]byte `toml:"array,omitempty"` + Slice []int `toml:"slice,omitempty"` + Map map[string]string `toml:"map,omitempty"` + } + + var v simple + encodeExpected(t, "fields with omitempty are omitted when empty", v, "", nil) + v = simple{ + Bool: true, + String: " ", + Slice: []int{2, 3, 4}, + Map: map[string]string{"foo": "bar"}, + } + expected := `bool = true +string = " " +slice = [2, 3, 4] + +[map] + foo = "bar" +` + encodeExpected(t, "fields with omitempty are not omitted when non-empty", + v, expected, nil) +} + +func TestEncodeWithOmitZero(t *testing.T) { + type simple struct { + Number int `toml:"number,omitzero"` + Real float64 `toml:"real,omitzero"` + Unsigned uint `toml:"unsigned,omitzero"` + } + + value := simple{0, 0.0, uint(0)} + expected := "" + + encodeExpected(t, "simple with omitzero, all zero", value, expected, nil) + + value.Number = 10 + value.Real = 20 + value.Unsigned = 5 + expected = `number = 10 +real = 20.0 +unsigned = 5 +` + encodeExpected(t, "simple with omitzero, non-zero", value, expected, nil) +} + +func TestEncodeOmitemptyWithEmptyName(t *testing.T) { + type simple struct { + S []int `toml:",omitempty"` + } + v := simple{[]int{1, 2, 3}} + expected := "S = [1, 2, 3]\n" + encodeExpected(t, "simple with omitempty, no name, non-empty field", + v, expected, nil) +} + +func TestEncodeAnonymousStruct(t *testing.T) { + type Inner struct{ N int } + type Outer0 struct{ Inner } + type Outer1 struct { + Inner `toml:"inner"` + } + + v0 := Outer0{Inner{3}} + expected := "N = 3\n" + encodeExpected(t, "embedded anonymous untagged struct", v0, expected, nil) + + v1 := Outer1{Inner{3}} + expected = "[inner]\n N = 3\n" + encodeExpected(t, "embedded anonymous tagged struct", v1, expected, nil) +} + +func TestEncodeAnonymousStructPointerField(t *testing.T) { + type Inner struct{ N int } + type Outer0 struct{ *Inner } + type Outer1 struct { + *Inner `toml:"inner"` + } + + v0 := Outer0{} + expected := "" + encodeExpected(t, "nil anonymous untagged struct pointer field", v0, expected, nil) + + v0 = Outer0{&Inner{3}} + expected = "N = 3\n" + encodeExpected(t, "non-nil anonymous untagged struct pointer field", v0, expected, nil) + + v1 := Outer1{} + expected = "" + encodeExpected(t, "nil anonymous tagged struct pointer field", v1, expected, nil) + + v1 = Outer1{&Inner{3}} + expected = "[inner]\n N = 3\n" + encodeExpected(t, "non-nil anonymous tagged struct pointer field", v1, expected, nil) +} + +func TestEncodeIgnoredFields(t *testing.T) { + type simple struct { + Number int `toml:"-"` + } + value := simple{} + expected := "" + encodeExpected(t, "ignored field", value, expected, nil) +} + +func encodeExpected( + t *testing.T, label string, val interface{}, wantStr string, wantErr error, +) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + err := enc.Encode(val) + if err != wantErr { + if wantErr != nil { + if wantErr == errAnything && err != nil { + return + } + t.Errorf("%s: want Encode error %v, got %v", label, wantErr, err) + } else { + t.Errorf("%s: Encode failed: %s", label, err) + } + } + if err != nil { + return + } + if got := buf.String(); wantStr != got { + t.Errorf("%s: want\n-----\n%q\n-----\nbut got\n-----\n%q\n-----\n", + label, wantStr, got) + } +} + +func ExampleEncoder_Encode() { + date, _ := time.Parse(time.RFC822, "14 Mar 10 18:00 UTC") + var config = map[string]interface{}{ + "date": date, + "counts": []int{1, 1, 2, 3, 5, 8}, + "hash": map[string]string{ + "key1": "val1", + "key2": "val2", + }, + } + buf := new(bytes.Buffer) + if err := NewEncoder(buf).Encode(config); err != nil { + log.Fatal(err) + } + fmt.Println(buf.String()) + + // Output: + // counts = [1, 1, 2, 3, 5, 8] + // date = 2010-03-14T18:00:00Z + // + // [hash] + // key1 = "val1" + // key2 = "val2" +} diff --git a/vendor/github.com/BurntSushi/toml/encoding_types.go b/vendor/github.com/BurntSushi/toml/encoding_types.go new file mode 100644 index 0000000..d36e1dd --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/encoding_types.go @@ -0,0 +1,19 @@ +// +build go1.2 + +package toml + +// In order to support Go 1.1, we define our own TextMarshaler and +// TextUnmarshaler types. For Go 1.2+, we just alias them with the +// standard library interfaces. + +import ( + "encoding" +) + +// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here +// so that Go 1.1 can be supported. +type TextMarshaler encoding.TextMarshaler + +// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined +// here so that Go 1.1 can be supported. +type TextUnmarshaler encoding.TextUnmarshaler diff --git a/vendor/github.com/BurntSushi/toml/encoding_types_1.1.go b/vendor/github.com/BurntSushi/toml/encoding_types_1.1.go new file mode 100644 index 0000000..e8d503d --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/encoding_types_1.1.go @@ -0,0 +1,18 @@ +// +build !go1.2 + +package toml + +// These interfaces were introduced in Go 1.2, so we add them manually when +// compiling for Go 1.1. + +// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here +// so that Go 1.1 can be supported. +type TextMarshaler interface { + MarshalText() (text []byte, err error) +} + +// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined +// here so that Go 1.1 can be supported. +type TextUnmarshaler interface { + UnmarshalText(text []byte) error +} diff --git a/vendor/github.com/BurntSushi/toml/lex.go b/vendor/github.com/BurntSushi/toml/lex.go new file mode 100644 index 0000000..e0a742a --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/lex.go @@ -0,0 +1,953 @@ +package toml + +import ( + "fmt" + "strings" + "unicode" + "unicode/utf8" +) + +type itemType int + +const ( + itemError itemType = iota + itemNIL // used in the parser to indicate no type + itemEOF + itemText + itemString + itemRawString + itemMultilineString + itemRawMultilineString + itemBool + itemInteger + itemFloat + itemDatetime + itemArray // the start of an array + itemArrayEnd + itemTableStart + itemTableEnd + itemArrayTableStart + itemArrayTableEnd + itemKeyStart + itemCommentStart + itemInlineTableStart + itemInlineTableEnd +) + +const ( + eof = 0 + comma = ',' + tableStart = '[' + tableEnd = ']' + arrayTableStart = '[' + arrayTableEnd = ']' + tableSep = '.' + keySep = '=' + arrayStart = '[' + arrayEnd = ']' + commentStart = '#' + stringStart = '"' + stringEnd = '"' + rawStringStart = '\'' + rawStringEnd = '\'' + inlineTableStart = '{' + inlineTableEnd = '}' +) + +type stateFn func(lx *lexer) stateFn + +type lexer struct { + input string + start int + pos int + line int + state stateFn + items chan item + + // Allow for backing up up to three runes. + // This is necessary because TOML contains 3-rune tokens (""" and '''). + prevWidths [3]int + nprev int // how many of prevWidths are in use + // If we emit an eof, we can still back up, but it is not OK to call + // next again. + atEOF bool + + // A stack of state functions used to maintain context. + // The idea is to reuse parts of the state machine in various places. + // For example, values can appear at the top level or within arbitrarily + // nested arrays. The last state on the stack is used after a value has + // been lexed. Similarly for comments. + stack []stateFn +} + +type item struct { + typ itemType + val string + line int +} + +func (lx *lexer) nextItem() item { + for { + select { + case item := <-lx.items: + return item + default: + lx.state = lx.state(lx) + } + } +} + +func lex(input string) *lexer { + lx := &lexer{ + input: input, + state: lexTop, + line: 1, + items: make(chan item, 10), + stack: make([]stateFn, 0, 10), + } + return lx +} + +func (lx *lexer) push(state stateFn) { + lx.stack = append(lx.stack, state) +} + +func (lx *lexer) pop() stateFn { + if len(lx.stack) == 0 { + return lx.errorf("BUG in lexer: no states to pop") + } + last := lx.stack[len(lx.stack)-1] + lx.stack = lx.stack[0 : len(lx.stack)-1] + return last +} + +func (lx *lexer) current() string { + return lx.input[lx.start:lx.pos] +} + +func (lx *lexer) emit(typ itemType) { + lx.items <- item{typ, lx.current(), lx.line} + lx.start = lx.pos +} + +func (lx *lexer) emitTrim(typ itemType) { + lx.items <- item{typ, strings.TrimSpace(lx.current()), lx.line} + lx.start = lx.pos +} + +func (lx *lexer) next() (r rune) { + if lx.atEOF { + panic("next called after EOF") + } + if lx.pos >= len(lx.input) { + lx.atEOF = true + return eof + } + + if lx.input[lx.pos] == '\n' { + lx.line++ + } + lx.prevWidths[2] = lx.prevWidths[1] + lx.prevWidths[1] = lx.prevWidths[0] + if lx.nprev < 3 { + lx.nprev++ + } + r, w := utf8.DecodeRuneInString(lx.input[lx.pos:]) + lx.prevWidths[0] = w + lx.pos += w + return r +} + +// ignore skips over the pending input before this point. +func (lx *lexer) ignore() { + lx.start = lx.pos +} + +// backup steps back one rune. Can be called only twice between calls to next. +func (lx *lexer) backup() { + if lx.atEOF { + lx.atEOF = false + return + } + if lx.nprev < 1 { + panic("backed up too far") + } + w := lx.prevWidths[0] + lx.prevWidths[0] = lx.prevWidths[1] + lx.prevWidths[1] = lx.prevWidths[2] + lx.nprev-- + lx.pos -= w + if lx.pos < len(lx.input) && lx.input[lx.pos] == '\n' { + lx.line-- + } +} + +// accept consumes the next rune if it's equal to `valid`. +func (lx *lexer) accept(valid rune) bool { + if lx.next() == valid { + return true + } + lx.backup() + return false +} + +// peek returns but does not consume the next rune in the input. +func (lx *lexer) peek() rune { + r := lx.next() + lx.backup() + return r +} + +// skip ignores all input that matches the given predicate. +func (lx *lexer) skip(pred func(rune) bool) { + for { + r := lx.next() + if pred(r) { + continue + } + lx.backup() + lx.ignore() + return + } +} + +// errorf stops all lexing by emitting an error and returning `nil`. +// Note that any value that is a character is escaped if it's a special +// character (newlines, tabs, etc.). +func (lx *lexer) errorf(format string, values ...interface{}) stateFn { + lx.items <- item{ + itemError, + fmt.Sprintf(format, values...), + lx.line, + } + return nil +} + +// lexTop consumes elements at the top level of TOML data. +func lexTop(lx *lexer) stateFn { + r := lx.next() + if isWhitespace(r) || isNL(r) { + return lexSkip(lx, lexTop) + } + switch r { + case commentStart: + lx.push(lexTop) + return lexCommentStart + case tableStart: + return lexTableStart + case eof: + if lx.pos > lx.start { + return lx.errorf("unexpected EOF") + } + lx.emit(itemEOF) + return nil + } + + // At this point, the only valid item can be a key, so we back up + // and let the key lexer do the rest. + lx.backup() + lx.push(lexTopEnd) + return lexKeyStart +} + +// lexTopEnd is entered whenever a top-level item has been consumed. (A value +// or a table.) It must see only whitespace, and will turn back to lexTop +// upon a newline. If it sees EOF, it will quit the lexer successfully. +func lexTopEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case r == commentStart: + // a comment will read to a newline for us. + lx.push(lexTop) + return lexCommentStart + case isWhitespace(r): + return lexTopEnd + case isNL(r): + lx.ignore() + return lexTop + case r == eof: + lx.emit(itemEOF) + return nil + } + return lx.errorf("expected a top-level item to end with a newline, "+ + "comment, or EOF, but got %q instead", r) +} + +// lexTable lexes the beginning of a table. Namely, it makes sure that +// it starts with a character other than '.' and ']'. +// It assumes that '[' has already been consumed. +// It also handles the case that this is an item in an array of tables. +// e.g., '[[name]]'. +func lexTableStart(lx *lexer) stateFn { + if lx.peek() == arrayTableStart { + lx.next() + lx.emit(itemArrayTableStart) + lx.push(lexArrayTableEnd) + } else { + lx.emit(itemTableStart) + lx.push(lexTableEnd) + } + return lexTableNameStart +} + +func lexTableEnd(lx *lexer) stateFn { + lx.emit(itemTableEnd) + return lexTopEnd +} + +func lexArrayTableEnd(lx *lexer) stateFn { + if r := lx.next(); r != arrayTableEnd { + return lx.errorf("expected end of table array name delimiter %q, "+ + "but got %q instead", arrayTableEnd, r) + } + lx.emit(itemArrayTableEnd) + return lexTopEnd +} + +func lexTableNameStart(lx *lexer) stateFn { + lx.skip(isWhitespace) + switch r := lx.peek(); { + case r == tableEnd || r == eof: + return lx.errorf("unexpected end of table name " + + "(table names cannot be empty)") + case r == tableSep: + return lx.errorf("unexpected table separator " + + "(table names cannot be empty)") + case r == stringStart || r == rawStringStart: + lx.ignore() + lx.push(lexTableNameEnd) + return lexValue // reuse string lexing + default: + return lexBareTableName + } +} + +// lexBareTableName lexes the name of a table. It assumes that at least one +// valid character for the table has already been read. +func lexBareTableName(lx *lexer) stateFn { + r := lx.next() + if isBareKeyChar(r) { + return lexBareTableName + } + lx.backup() + lx.emit(itemText) + return lexTableNameEnd +} + +// lexTableNameEnd reads the end of a piece of a table name, optionally +// consuming whitespace. +func lexTableNameEnd(lx *lexer) stateFn { + lx.skip(isWhitespace) + switch r := lx.next(); { + case isWhitespace(r): + return lexTableNameEnd + case r == tableSep: + lx.ignore() + return lexTableNameStart + case r == tableEnd: + return lx.pop() + default: + return lx.errorf("expected '.' or ']' to end table name, "+ + "but got %q instead", r) + } +} + +// lexKeyStart consumes a key name up until the first non-whitespace character. +// lexKeyStart will ignore whitespace. +func lexKeyStart(lx *lexer) stateFn { + r := lx.peek() + switch { + case r == keySep: + return lx.errorf("unexpected key separator %q", keySep) + case isWhitespace(r) || isNL(r): + lx.next() + return lexSkip(lx, lexKeyStart) + case r == stringStart || r == rawStringStart: + lx.ignore() + lx.emit(itemKeyStart) + lx.push(lexKeyEnd) + return lexValue // reuse string lexing + default: + lx.ignore() + lx.emit(itemKeyStart) + return lexBareKey + } +} + +// lexBareKey consumes the text of a bare key. Assumes that the first character +// (which is not whitespace) has not yet been consumed. +func lexBareKey(lx *lexer) stateFn { + switch r := lx.next(); { + case isBareKeyChar(r): + return lexBareKey + case isWhitespace(r): + lx.backup() + lx.emit(itemText) + return lexKeyEnd + case r == keySep: + lx.backup() + lx.emit(itemText) + return lexKeyEnd + default: + return lx.errorf("bare keys cannot contain %q", r) + } +} + +// lexKeyEnd consumes the end of a key and trims whitespace (up to the key +// separator). +func lexKeyEnd(lx *lexer) stateFn { + switch r := lx.next(); { + case r == keySep: + return lexSkip(lx, lexValue) + case isWhitespace(r): + return lexSkip(lx, lexKeyEnd) + default: + return lx.errorf("expected key separator %q, but got %q instead", + keySep, r) + } +} + +// lexValue starts the consumption of a value anywhere a value is expected. +// lexValue will ignore whitespace. +// After a value is lexed, the last state on the next is popped and returned. +func lexValue(lx *lexer) stateFn { + // We allow whitespace to precede a value, but NOT newlines. + // In array syntax, the array states are responsible for ignoring newlines. + r := lx.next() + switch { + case isWhitespace(r): + return lexSkip(lx, lexValue) + case isDigit(r): + lx.backup() // avoid an extra state and use the same as above + return lexNumberOrDateStart + } + switch r { + case arrayStart: + lx.ignore() + lx.emit(itemArray) + return lexArrayValue + case inlineTableStart: + lx.ignore() + lx.emit(itemInlineTableStart) + return lexInlineTableValue + case stringStart: + if lx.accept(stringStart) { + if lx.accept(stringStart) { + lx.ignore() // Ignore """ + return lexMultilineString + } + lx.backup() + } + lx.ignore() // ignore the '"' + return lexString + case rawStringStart: + if lx.accept(rawStringStart) { + if lx.accept(rawStringStart) { + lx.ignore() // Ignore """ + return lexMultilineRawString + } + lx.backup() + } + lx.ignore() // ignore the "'" + return lexRawString + case '+', '-': + return lexNumberStart + case '.': // special error case, be kind to users + return lx.errorf("floats must start with a digit, not '.'") + } + if unicode.IsLetter(r) { + // Be permissive here; lexBool will give a nice error if the + // user wrote something like + // x = foo + // (i.e. not 'true' or 'false' but is something else word-like.) + lx.backup() + return lexBool + } + return lx.errorf("expected value but found %q instead", r) +} + +// lexArrayValue consumes one value in an array. It assumes that '[' or ',' +// have already been consumed. All whitespace and newlines are ignored. +func lexArrayValue(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r) || isNL(r): + return lexSkip(lx, lexArrayValue) + case r == commentStart: + lx.push(lexArrayValue) + return lexCommentStart + case r == comma: + return lx.errorf("unexpected comma") + case r == arrayEnd: + // NOTE(caleb): The spec isn't clear about whether you can have + // a trailing comma or not, so we'll allow it. + return lexArrayEnd + } + + lx.backup() + lx.push(lexArrayValueEnd) + return lexValue +} + +// lexArrayValueEnd consumes everything between the end of an array value and +// the next value (or the end of the array): it ignores whitespace and newlines +// and expects either a ',' or a ']'. +func lexArrayValueEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r) || isNL(r): + return lexSkip(lx, lexArrayValueEnd) + case r == commentStart: + lx.push(lexArrayValueEnd) + return lexCommentStart + case r == comma: + lx.ignore() + return lexArrayValue // move on to the next value + case r == arrayEnd: + return lexArrayEnd + } + return lx.errorf( + "expected a comma or array terminator %q, but got %q instead", + arrayEnd, r, + ) +} + +// lexArrayEnd finishes the lexing of an array. +// It assumes that a ']' has just been consumed. +func lexArrayEnd(lx *lexer) stateFn { + lx.ignore() + lx.emit(itemArrayEnd) + return lx.pop() +} + +// lexInlineTableValue consumes one key/value pair in an inline table. +// It assumes that '{' or ',' have already been consumed. Whitespace is ignored. +func lexInlineTableValue(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r): + return lexSkip(lx, lexInlineTableValue) + case isNL(r): + return lx.errorf("newlines not allowed within inline tables") + case r == commentStart: + lx.push(lexInlineTableValue) + return lexCommentStart + case r == comma: + return lx.errorf("unexpected comma") + case r == inlineTableEnd: + return lexInlineTableEnd + } + lx.backup() + lx.push(lexInlineTableValueEnd) + return lexKeyStart +} + +// lexInlineTableValueEnd consumes everything between the end of an inline table +// key/value pair and the next pair (or the end of the table): +// it ignores whitespace and expects either a ',' or a '}'. +func lexInlineTableValueEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r): + return lexSkip(lx, lexInlineTableValueEnd) + case isNL(r): + return lx.errorf("newlines not allowed within inline tables") + case r == commentStart: + lx.push(lexInlineTableValueEnd) + return lexCommentStart + case r == comma: + lx.ignore() + return lexInlineTableValue + case r == inlineTableEnd: + return lexInlineTableEnd + } + return lx.errorf("expected a comma or an inline table terminator %q, "+ + "but got %q instead", inlineTableEnd, r) +} + +// lexInlineTableEnd finishes the lexing of an inline table. +// It assumes that a '}' has just been consumed. +func lexInlineTableEnd(lx *lexer) stateFn { + lx.ignore() + lx.emit(itemInlineTableEnd) + return lx.pop() +} + +// lexString consumes the inner contents of a string. It assumes that the +// beginning '"' has already been consumed and ignored. +func lexString(lx *lexer) stateFn { + r := lx.next() + switch { + case r == eof: + return lx.errorf("unexpected EOF") + case isNL(r): + return lx.errorf("strings cannot contain newlines") + case r == '\\': + lx.push(lexString) + return lexStringEscape + case r == stringEnd: + lx.backup() + lx.emit(itemString) + lx.next() + lx.ignore() + return lx.pop() + } + return lexString +} + +// lexMultilineString consumes the inner contents of a string. It assumes that +// the beginning '"""' has already been consumed and ignored. +func lexMultilineString(lx *lexer) stateFn { + switch lx.next() { + case eof: + return lx.errorf("unexpected EOF") + case '\\': + return lexMultilineStringEscape + case stringEnd: + if lx.accept(stringEnd) { + if lx.accept(stringEnd) { + lx.backup() + lx.backup() + lx.backup() + lx.emit(itemMultilineString) + lx.next() + lx.next() + lx.next() + lx.ignore() + return lx.pop() + } + lx.backup() + } + } + return lexMultilineString +} + +// lexRawString consumes a raw string. Nothing can be escaped in such a string. +// It assumes that the beginning "'" has already been consumed and ignored. +func lexRawString(lx *lexer) stateFn { + r := lx.next() + switch { + case r == eof: + return lx.errorf("unexpected EOF") + case isNL(r): + return lx.errorf("strings cannot contain newlines") + case r == rawStringEnd: + lx.backup() + lx.emit(itemRawString) + lx.next() + lx.ignore() + return lx.pop() + } + return lexRawString +} + +// lexMultilineRawString consumes a raw string. Nothing can be escaped in such +// a string. It assumes that the beginning "'''" has already been consumed and +// ignored. +func lexMultilineRawString(lx *lexer) stateFn { + switch lx.next() { + case eof: + return lx.errorf("unexpected EOF") + case rawStringEnd: + if lx.accept(rawStringEnd) { + if lx.accept(rawStringEnd) { + lx.backup() + lx.backup() + lx.backup() + lx.emit(itemRawMultilineString) + lx.next() + lx.next() + lx.next() + lx.ignore() + return lx.pop() + } + lx.backup() + } + } + return lexMultilineRawString +} + +// lexMultilineStringEscape consumes an escaped character. It assumes that the +// preceding '\\' has already been consumed. +func lexMultilineStringEscape(lx *lexer) stateFn { + // Handle the special case first: + if isNL(lx.next()) { + return lexMultilineString + } + lx.backup() + lx.push(lexMultilineString) + return lexStringEscape(lx) +} + +func lexStringEscape(lx *lexer) stateFn { + r := lx.next() + switch r { + case 'b': + fallthrough + case 't': + fallthrough + case 'n': + fallthrough + case 'f': + fallthrough + case 'r': + fallthrough + case '"': + fallthrough + case '\\': + return lx.pop() + case 'u': + return lexShortUnicodeEscape + case 'U': + return lexLongUnicodeEscape + } + return lx.errorf("invalid escape character %q; only the following "+ + "escape characters are allowed: "+ + `\b, \t, \n, \f, \r, \", \\, \uXXXX, and \UXXXXXXXX`, r) +} + +func lexShortUnicodeEscape(lx *lexer) stateFn { + var r rune + for i := 0; i < 4; i++ { + r = lx.next() + if !isHexadecimal(r) { + return lx.errorf(`expected four hexadecimal digits after '\u', `+ + "but got %q instead", lx.current()) + } + } + return lx.pop() +} + +func lexLongUnicodeEscape(lx *lexer) stateFn { + var r rune + for i := 0; i < 8; i++ { + r = lx.next() + if !isHexadecimal(r) { + return lx.errorf(`expected eight hexadecimal digits after '\U', `+ + "but got %q instead", lx.current()) + } + } + return lx.pop() +} + +// lexNumberOrDateStart consumes either an integer, a float, or datetime. +func lexNumberOrDateStart(lx *lexer) stateFn { + r := lx.next() + if isDigit(r) { + return lexNumberOrDate + } + switch r { + case '_': + return lexNumber + case 'e', 'E': + return lexFloat + case '.': + return lx.errorf("floats must start with a digit, not '.'") + } + return lx.errorf("expected a digit but got %q", r) +} + +// lexNumberOrDate consumes either an integer, float or datetime. +func lexNumberOrDate(lx *lexer) stateFn { + r := lx.next() + if isDigit(r) { + return lexNumberOrDate + } + switch r { + case '-': + return lexDatetime + case '_': + return lexNumber + case '.', 'e', 'E': + return lexFloat + } + + lx.backup() + lx.emit(itemInteger) + return lx.pop() +} + +// lexDatetime consumes a Datetime, to a first approximation. +// The parser validates that it matches one of the accepted formats. +func lexDatetime(lx *lexer) stateFn { + r := lx.next() + if isDigit(r) { + return lexDatetime + } + switch r { + case '-', 'T', ':', '.', 'Z', '+': + return lexDatetime + } + + lx.backup() + lx.emit(itemDatetime) + return lx.pop() +} + +// lexNumberStart consumes either an integer or a float. It assumes that a sign +// has already been read, but that *no* digits have been consumed. +// lexNumberStart will move to the appropriate integer or float states. +func lexNumberStart(lx *lexer) stateFn { + // We MUST see a digit. Even floats have to start with a digit. + r := lx.next() + if !isDigit(r) { + if r == '.' { + return lx.errorf("floats must start with a digit, not '.'") + } + return lx.errorf("expected a digit but got %q", r) + } + return lexNumber +} + +// lexNumber consumes an integer or a float after seeing the first digit. +func lexNumber(lx *lexer) stateFn { + r := lx.next() + if isDigit(r) { + return lexNumber + } + switch r { + case '_': + return lexNumber + case '.', 'e', 'E': + return lexFloat + } + + lx.backup() + lx.emit(itemInteger) + return lx.pop() +} + +// lexFloat consumes the elements of a float. It allows any sequence of +// float-like characters, so floats emitted by the lexer are only a first +// approximation and must be validated by the parser. +func lexFloat(lx *lexer) stateFn { + r := lx.next() + if isDigit(r) { + return lexFloat + } + switch r { + case '_', '.', '-', '+', 'e', 'E': + return lexFloat + } + + lx.backup() + lx.emit(itemFloat) + return lx.pop() +} + +// lexBool consumes a bool string: 'true' or 'false. +func lexBool(lx *lexer) stateFn { + var rs []rune + for { + r := lx.next() + if !unicode.IsLetter(r) { + lx.backup() + break + } + rs = append(rs, r) + } + s := string(rs) + switch s { + case "true", "false": + lx.emit(itemBool) + return lx.pop() + } + return lx.errorf("expected value but found %q instead", s) +} + +// lexCommentStart begins the lexing of a comment. It will emit +// itemCommentStart and consume no characters, passing control to lexComment. +func lexCommentStart(lx *lexer) stateFn { + lx.ignore() + lx.emit(itemCommentStart) + return lexComment +} + +// lexComment lexes an entire comment. It assumes that '#' has been consumed. +// It will consume *up to* the first newline character, and pass control +// back to the last state on the stack. +func lexComment(lx *lexer) stateFn { + r := lx.peek() + if isNL(r) || r == eof { + lx.emit(itemText) + return lx.pop() + } + lx.next() + return lexComment +} + +// lexSkip ignores all slurped input and moves on to the next state. +func lexSkip(lx *lexer, nextState stateFn) stateFn { + return func(lx *lexer) stateFn { + lx.ignore() + return nextState + } +} + +// isWhitespace returns true if `r` is a whitespace character according +// to the spec. +func isWhitespace(r rune) bool { + return r == '\t' || r == ' ' +} + +func isNL(r rune) bool { + return r == '\n' || r == '\r' +} + +func isDigit(r rune) bool { + return r >= '0' && r <= '9' +} + +func isHexadecimal(r rune) bool { + return (r >= '0' && r <= '9') || + (r >= 'a' && r <= 'f') || + (r >= 'A' && r <= 'F') +} + +func isBareKeyChar(r rune) bool { + return (r >= 'A' && r <= 'Z') || + (r >= 'a' && r <= 'z') || + (r >= '0' && r <= '9') || + r == '_' || + r == '-' +} + +func (itype itemType) String() string { + switch itype { + case itemError: + return "Error" + case itemNIL: + return "NIL" + case itemEOF: + return "EOF" + case itemText: + return "Text" + case itemString, itemRawString, itemMultilineString, itemRawMultilineString: + return "String" + case itemBool: + return "Bool" + case itemInteger: + return "Integer" + case itemFloat: + return "Float" + case itemDatetime: + return "DateTime" + case itemTableStart: + return "TableStart" + case itemTableEnd: + return "TableEnd" + case itemKeyStart: + return "KeyStart" + case itemArray: + return "Array" + case itemArrayEnd: + return "ArrayEnd" + case itemCommentStart: + return "CommentStart" + } + panic(fmt.Sprintf("BUG: Unknown type '%d'.", int(itype))) +} + +func (item item) String() string { + return fmt.Sprintf("(%s, %s)", item.typ.String(), item.val) +} diff --git a/vendor/github.com/BurntSushi/toml/parse.go b/vendor/github.com/BurntSushi/toml/parse.go new file mode 100644 index 0000000..50869ef --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/parse.go @@ -0,0 +1,592 @@ +package toml + +import ( + "fmt" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" +) + +type parser struct { + mapping map[string]interface{} + types map[string]tomlType + lx *lexer + + // A list of keys in the order that they appear in the TOML data. + ordered []Key + + // the full key for the current hash in scope + context Key + + // the base key name for everything except hashes + currentKey string + + // rough approximation of line number + approxLine int + + // A map of 'key.group.names' to whether they were created implicitly. + implicits map[string]bool +} + +type parseError string + +func (pe parseError) Error() string { + return string(pe) +} + +func parse(data string) (p *parser, err error) { + defer func() { + if r := recover(); r != nil { + var ok bool + if err, ok = r.(parseError); ok { + return + } + panic(r) + } + }() + + p = &parser{ + mapping: make(map[string]interface{}), + types: make(map[string]tomlType), + lx: lex(data), + ordered: make([]Key, 0), + implicits: make(map[string]bool), + } + for { + item := p.next() + if item.typ == itemEOF { + break + } + p.topLevel(item) + } + + return p, nil +} + +func (p *parser) panicf(format string, v ...interface{}) { + msg := fmt.Sprintf("Near line %d (last key parsed '%s'): %s", + p.approxLine, p.current(), fmt.Sprintf(format, v...)) + panic(parseError(msg)) +} + +func (p *parser) next() item { + it := p.lx.nextItem() + if it.typ == itemError { + p.panicf("%s", it.val) + } + return it +} + +func (p *parser) bug(format string, v ...interface{}) { + panic(fmt.Sprintf("BUG: "+format+"\n\n", v...)) +} + +func (p *parser) expect(typ itemType) item { + it := p.next() + p.assertEqual(typ, it.typ) + return it +} + +func (p *parser) assertEqual(expected, got itemType) { + if expected != got { + p.bug("Expected '%s' but got '%s'.", expected, got) + } +} + +func (p *parser) topLevel(item item) { + switch item.typ { + case itemCommentStart: + p.approxLine = item.line + p.expect(itemText) + case itemTableStart: + kg := p.next() + p.approxLine = kg.line + + var key Key + for ; kg.typ != itemTableEnd && kg.typ != itemEOF; kg = p.next() { + key = append(key, p.keyString(kg)) + } + p.assertEqual(itemTableEnd, kg.typ) + + p.establishContext(key, false) + p.setType("", tomlHash) + p.ordered = append(p.ordered, key) + case itemArrayTableStart: + kg := p.next() + p.approxLine = kg.line + + var key Key + for ; kg.typ != itemArrayTableEnd && kg.typ != itemEOF; kg = p.next() { + key = append(key, p.keyString(kg)) + } + p.assertEqual(itemArrayTableEnd, kg.typ) + + p.establishContext(key, true) + p.setType("", tomlArrayHash) + p.ordered = append(p.ordered, key) + case itemKeyStart: + kname := p.next() + p.approxLine = kname.line + p.currentKey = p.keyString(kname) + + val, typ := p.value(p.next()) + p.setValue(p.currentKey, val) + p.setType(p.currentKey, typ) + p.ordered = append(p.ordered, p.context.add(p.currentKey)) + p.currentKey = "" + default: + p.bug("Unexpected type at top level: %s", item.typ) + } +} + +// Gets a string for a key (or part of a key in a table name). +func (p *parser) keyString(it item) string { + switch it.typ { + case itemText: + return it.val + case itemString, itemMultilineString, + itemRawString, itemRawMultilineString: + s, _ := p.value(it) + return s.(string) + default: + p.bug("Unexpected key type: %s", it.typ) + panic("unreachable") + } +} + +// value translates an expected value from the lexer into a Go value wrapped +// as an empty interface. +func (p *parser) value(it item) (interface{}, tomlType) { + switch it.typ { + case itemString: + return p.replaceEscapes(it.val), p.typeOfPrimitive(it) + case itemMultilineString: + trimmed := stripFirstNewline(stripEscapedWhitespace(it.val)) + return p.replaceEscapes(trimmed), p.typeOfPrimitive(it) + case itemRawString: + return it.val, p.typeOfPrimitive(it) + case itemRawMultilineString: + return stripFirstNewline(it.val), p.typeOfPrimitive(it) + case itemBool: + switch it.val { + case "true": + return true, p.typeOfPrimitive(it) + case "false": + return false, p.typeOfPrimitive(it) + } + p.bug("Expected boolean value, but got '%s'.", it.val) + case itemInteger: + if !numUnderscoresOK(it.val) { + p.panicf("Invalid integer %q: underscores must be surrounded by digits", + it.val) + } + val := strings.Replace(it.val, "_", "", -1) + num, err := strconv.ParseInt(val, 10, 64) + if err != nil { + // Distinguish integer values. Normally, it'd be a bug if the lexer + // provides an invalid integer, but it's possible that the number is + // out of range of valid values (which the lexer cannot determine). + // So mark the former as a bug but the latter as a legitimate user + // error. + if e, ok := err.(*strconv.NumError); ok && + e.Err == strconv.ErrRange { + + p.panicf("Integer '%s' is out of the range of 64-bit "+ + "signed integers.", it.val) + } else { + p.bug("Expected integer value, but got '%s'.", it.val) + } + } + return num, p.typeOfPrimitive(it) + case itemFloat: + parts := strings.FieldsFunc(it.val, func(r rune) bool { + switch r { + case '.', 'e', 'E': + return true + } + return false + }) + for _, part := range parts { + if !numUnderscoresOK(part) { + p.panicf("Invalid float %q: underscores must be "+ + "surrounded by digits", it.val) + } + } + if !numPeriodsOK(it.val) { + // As a special case, numbers like '123.' or '1.e2', + // which are valid as far as Go/strconv are concerned, + // must be rejected because TOML says that a fractional + // part consists of '.' followed by 1+ digits. + p.panicf("Invalid float %q: '.' must be followed "+ + "by one or more digits", it.val) + } + val := strings.Replace(it.val, "_", "", -1) + num, err := strconv.ParseFloat(val, 64) + if err != nil { + if e, ok := err.(*strconv.NumError); ok && + e.Err == strconv.ErrRange { + + p.panicf("Float '%s' is out of the range of 64-bit "+ + "IEEE-754 floating-point numbers.", it.val) + } else { + p.panicf("Invalid float value: %q", it.val) + } + } + return num, p.typeOfPrimitive(it) + case itemDatetime: + var t time.Time + var ok bool + var err error + for _, format := range []string{ + "2006-01-02T15:04:05Z07:00", + "2006-01-02T15:04:05", + "2006-01-02", + } { + t, err = time.ParseInLocation(format, it.val, time.Local) + if err == nil { + ok = true + break + } + } + if !ok { + p.panicf("Invalid TOML Datetime: %q.", it.val) + } + return t, p.typeOfPrimitive(it) + case itemArray: + array := make([]interface{}, 0) + types := make([]tomlType, 0) + + for it = p.next(); it.typ != itemArrayEnd; it = p.next() { + if it.typ == itemCommentStart { + p.expect(itemText) + continue + } + + val, typ := p.value(it) + array = append(array, val) + types = append(types, typ) + } + return array, p.typeOfArray(types) + case itemInlineTableStart: + var ( + hash = make(map[string]interface{}) + outerContext = p.context + outerKey = p.currentKey + ) + + p.context = append(p.context, p.currentKey) + p.currentKey = "" + for it := p.next(); it.typ != itemInlineTableEnd; it = p.next() { + if it.typ != itemKeyStart { + p.bug("Expected key start but instead found %q, around line %d", + it.val, p.approxLine) + } + if it.typ == itemCommentStart { + p.expect(itemText) + continue + } + + // retrieve key + k := p.next() + p.approxLine = k.line + kname := p.keyString(k) + + // retrieve value + p.currentKey = kname + val, typ := p.value(p.next()) + // make sure we keep metadata up to date + p.setType(kname, typ) + p.ordered = append(p.ordered, p.context.add(p.currentKey)) + hash[kname] = val + } + p.context = outerContext + p.currentKey = outerKey + return hash, tomlHash + } + p.bug("Unexpected value type: %s", it.typ) + panic("unreachable") +} + +// numUnderscoresOK checks whether each underscore in s is surrounded by +// characters that are not underscores. +func numUnderscoresOK(s string) bool { + accept := false + for _, r := range s { + if r == '_' { + if !accept { + return false + } + accept = false + continue + } + accept = true + } + return accept +} + +// numPeriodsOK checks whether every period in s is followed by a digit. +func numPeriodsOK(s string) bool { + period := false + for _, r := range s { + if period && !isDigit(r) { + return false + } + period = r == '.' + } + return !period +} + +// establishContext sets the current context of the parser, +// where the context is either a hash or an array of hashes. Which one is +// set depends on the value of the `array` parameter. +// +// Establishing the context also makes sure that the key isn't a duplicate, and +// will create implicit hashes automatically. +func (p *parser) establishContext(key Key, array bool) { + var ok bool + + // Always start at the top level and drill down for our context. + hashContext := p.mapping + keyContext := make(Key, 0) + + // We only need implicit hashes for key[0:-1] + for _, k := range key[0 : len(key)-1] { + _, ok = hashContext[k] + keyContext = append(keyContext, k) + + // No key? Make an implicit hash and move on. + if !ok { + p.addImplicit(keyContext) + hashContext[k] = make(map[string]interface{}) + } + + // If the hash context is actually an array of tables, then set + // the hash context to the last element in that array. + // + // Otherwise, it better be a table, since this MUST be a key group (by + // virtue of it not being the last element in a key). + switch t := hashContext[k].(type) { + case []map[string]interface{}: + hashContext = t[len(t)-1] + case map[string]interface{}: + hashContext = t + default: + p.panicf("Key '%s' was already created as a hash.", keyContext) + } + } + + p.context = keyContext + if array { + // If this is the first element for this array, then allocate a new + // list of tables for it. + k := key[len(key)-1] + if _, ok := hashContext[k]; !ok { + hashContext[k] = make([]map[string]interface{}, 0, 5) + } + + // Add a new table. But make sure the key hasn't already been used + // for something else. + if hash, ok := hashContext[k].([]map[string]interface{}); ok { + hashContext[k] = append(hash, make(map[string]interface{})) + } else { + p.panicf("Key '%s' was already created and cannot be used as "+ + "an array.", keyContext) + } + } else { + p.setValue(key[len(key)-1], make(map[string]interface{})) + } + p.context = append(p.context, key[len(key)-1]) +} + +// setValue sets the given key to the given value in the current context. +// It will make sure that the key hasn't already been defined, account for +// implicit key groups. +func (p *parser) setValue(key string, value interface{}) { + var tmpHash interface{} + var ok bool + + hash := p.mapping + keyContext := make(Key, 0) + for _, k := range p.context { + keyContext = append(keyContext, k) + if tmpHash, ok = hash[k]; !ok { + p.bug("Context for key '%s' has not been established.", keyContext) + } + switch t := tmpHash.(type) { + case []map[string]interface{}: + // The context is a table of hashes. Pick the most recent table + // defined as the current hash. + hash = t[len(t)-1] + case map[string]interface{}: + hash = t + default: + p.bug("Expected hash to have type 'map[string]interface{}', but "+ + "it has '%T' instead.", tmpHash) + } + } + keyContext = append(keyContext, key) + + if _, ok := hash[key]; ok { + // Typically, if the given key has already been set, then we have + // to raise an error since duplicate keys are disallowed. However, + // it's possible that a key was previously defined implicitly. In this + // case, it is allowed to be redefined concretely. (See the + // `tests/valid/implicit-and-explicit-after.toml` test in `toml-test`.) + // + // But we have to make sure to stop marking it as an implicit. (So that + // another redefinition provokes an error.) + // + // Note that since it has already been defined (as a hash), we don't + // want to overwrite it. So our business is done. + if p.isImplicit(keyContext) { + p.removeImplicit(keyContext) + return + } + + // Otherwise, we have a concrete key trying to override a previous + // key, which is *always* wrong. + p.panicf("Key '%s' has already been defined.", keyContext) + } + hash[key] = value +} + +// setType sets the type of a particular value at a given key. +// It should be called immediately AFTER setValue. +// +// Note that if `key` is empty, then the type given will be applied to the +// current context (which is either a table or an array of tables). +func (p *parser) setType(key string, typ tomlType) { + keyContext := make(Key, 0, len(p.context)+1) + for _, k := range p.context { + keyContext = append(keyContext, k) + } + if len(key) > 0 { // allow type setting for hashes + keyContext = append(keyContext, key) + } + p.types[keyContext.String()] = typ +} + +// addImplicit sets the given Key as having been created implicitly. +func (p *parser) addImplicit(key Key) { + p.implicits[key.String()] = true +} + +// removeImplicit stops tagging the given key as having been implicitly +// created. +func (p *parser) removeImplicit(key Key) { + p.implicits[key.String()] = false +} + +// isImplicit returns true if the key group pointed to by the key was created +// implicitly. +func (p *parser) isImplicit(key Key) bool { + return p.implicits[key.String()] +} + +// current returns the full key name of the current context. +func (p *parser) current() string { + if len(p.currentKey) == 0 { + return p.context.String() + } + if len(p.context) == 0 { + return p.currentKey + } + return fmt.Sprintf("%s.%s", p.context, p.currentKey) +} + +func stripFirstNewline(s string) string { + if len(s) == 0 || s[0] != '\n' { + return s + } + return s[1:] +} + +func stripEscapedWhitespace(s string) string { + esc := strings.Split(s, "\\\n") + if len(esc) > 1 { + for i := 1; i < len(esc); i++ { + esc[i] = strings.TrimLeftFunc(esc[i], unicode.IsSpace) + } + } + return strings.Join(esc, "") +} + +func (p *parser) replaceEscapes(str string) string { + var replaced []rune + s := []byte(str) + r := 0 + for r < len(s) { + if s[r] != '\\' { + c, size := utf8.DecodeRune(s[r:]) + r += size + replaced = append(replaced, c) + continue + } + r += 1 + if r >= len(s) { + p.bug("Escape sequence at end of string.") + return "" + } + switch s[r] { + default: + p.bug("Expected valid escape code after \\, but got %q.", s[r]) + return "" + case 'b': + replaced = append(replaced, rune(0x0008)) + r += 1 + case 't': + replaced = append(replaced, rune(0x0009)) + r += 1 + case 'n': + replaced = append(replaced, rune(0x000A)) + r += 1 + case 'f': + replaced = append(replaced, rune(0x000C)) + r += 1 + case 'r': + replaced = append(replaced, rune(0x000D)) + r += 1 + case '"': + replaced = append(replaced, rune(0x0022)) + r += 1 + case '\\': + replaced = append(replaced, rune(0x005C)) + r += 1 + case 'u': + // At this point, we know we have a Unicode escape of the form + // `uXXXX` at [r, r+5). (Because the lexer guarantees this + // for us.) + escaped := p.asciiEscapeToUnicode(s[r+1 : r+5]) + replaced = append(replaced, escaped) + r += 5 + case 'U': + // At this point, we know we have a Unicode escape of the form + // `uXXXX` at [r, r+9). (Because the lexer guarantees this + // for us.) + escaped := p.asciiEscapeToUnicode(s[r+1 : r+9]) + replaced = append(replaced, escaped) + r += 9 + } + } + return string(replaced) +} + +func (p *parser) asciiEscapeToUnicode(bs []byte) rune { + s := string(bs) + hex, err := strconv.ParseUint(strings.ToLower(s), 16, 32) + if err != nil { + p.bug("Could not parse '%s' as a hexadecimal number, but the "+ + "lexer claims it's OK: %s", s, err) + } + if !utf8.ValidRune(rune(hex)) { + p.panicf("Escaped character '\\u%s' is not valid UTF-8.", s) + } + return rune(hex) +} + +func isStringType(ty itemType) bool { + return ty == itemString || ty == itemMultilineString || + ty == itemRawString || ty == itemRawMultilineString +} diff --git a/vendor/github.com/BurntSushi/toml/session.vim b/vendor/github.com/BurntSushi/toml/session.vim new file mode 100644 index 0000000..562164b --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/session.vim @@ -0,0 +1 @@ +au BufWritePost *.go silent!make tags > /dev/null 2>&1 diff --git a/vendor/github.com/BurntSushi/toml/type_check.go b/vendor/github.com/BurntSushi/toml/type_check.go new file mode 100644 index 0000000..c73f8af --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/type_check.go @@ -0,0 +1,91 @@ +package toml + +// tomlType represents any Go type that corresponds to a TOML type. +// While the first draft of the TOML spec has a simplistic type system that +// probably doesn't need this level of sophistication, we seem to be militating +// toward adding real composite types. +type tomlType interface { + typeString() string +} + +// typeEqual accepts any two types and returns true if they are equal. +func typeEqual(t1, t2 tomlType) bool { + if t1 == nil || t2 == nil { + return false + } + return t1.typeString() == t2.typeString() +} + +func typeIsHash(t tomlType) bool { + return typeEqual(t, tomlHash) || typeEqual(t, tomlArrayHash) +} + +type tomlBaseType string + +func (btype tomlBaseType) typeString() string { + return string(btype) +} + +func (btype tomlBaseType) String() string { + return btype.typeString() +} + +var ( + tomlInteger tomlBaseType = "Integer" + tomlFloat tomlBaseType = "Float" + tomlDatetime tomlBaseType = "Datetime" + tomlString tomlBaseType = "String" + tomlBool tomlBaseType = "Bool" + tomlArray tomlBaseType = "Array" + tomlHash tomlBaseType = "Hash" + tomlArrayHash tomlBaseType = "ArrayHash" +) + +// typeOfPrimitive returns a tomlType of any primitive value in TOML. +// Primitive values are: Integer, Float, Datetime, String and Bool. +// +// Passing a lexer item other than the following will cause a BUG message +// to occur: itemString, itemBool, itemInteger, itemFloat, itemDatetime. +func (p *parser) typeOfPrimitive(lexItem item) tomlType { + switch lexItem.typ { + case itemInteger: + return tomlInteger + case itemFloat: + return tomlFloat + case itemDatetime: + return tomlDatetime + case itemString: + return tomlString + case itemMultilineString: + return tomlString + case itemRawString: + return tomlString + case itemRawMultilineString: + return tomlString + case itemBool: + return tomlBool + } + p.bug("Cannot infer primitive type of lex item '%s'.", lexItem) + panic("unreachable") +} + +// typeOfArray returns a tomlType for an array given a list of types of its +// values. +// +// In the current spec, if an array is homogeneous, then its type is always +// "Array". If the array is not homogeneous, an error is generated. +func (p *parser) typeOfArray(types []tomlType) tomlType { + // Empty arrays are cool. + if len(types) == 0 { + return tomlArray + } + + theType := types[0] + for _, t := range types[1:] { + if !typeEqual(theType, t) { + p.panicf("Array contains values of type '%s' and '%s', but "+ + "arrays must be homogeneous.", theType, t) + } + } + return tomlArray +} diff --git a/vendor/github.com/BurntSushi/toml/type_fields.go b/vendor/github.com/BurntSushi/toml/type_fields.go new file mode 100644 index 0000000..608997c --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/type_fields.go @@ -0,0 +1,242 @@ +package toml + +// Struct field handling is adapted from code in encoding/json: +// +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the Go distribution. + +import ( + "reflect" + "sort" + "sync" +) + +// A field represents a single field found in a struct. +type field struct { + name string // the name of the field (`toml` tag included) + tag bool // whether field has a `toml` tag + index []int // represents the depth of an anonymous field + typ reflect.Type // the type of the field +} + +// byName sorts field by name, breaking ties with depth, +// then breaking ties with "name came from toml tag", then +// breaking ties with index sequence. +type byName []field + +func (x byName) Len() int { return len(x) } + +func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byName) Less(i, j int) bool { + if x[i].name != x[j].name { + return x[i].name < x[j].name + } + if len(x[i].index) != len(x[j].index) { + return len(x[i].index) < len(x[j].index) + } + if x[i].tag != x[j].tag { + return x[i].tag + } + return byIndex(x).Less(i, j) +} + +// byIndex sorts field by index sequence. +type byIndex []field + +func (x byIndex) Len() int { return len(x) } + +func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byIndex) Less(i, j int) bool { + for k, xik := range x[i].index { + if k >= len(x[j].index) { + return false + } + if xik != x[j].index[k] { + return xik < x[j].index[k] + } + } + return len(x[i].index) < len(x[j].index) +} + +// typeFields returns a list of fields that TOML should recognize for the given +// type. The algorithm is breadth-first search over the set of structs to +// include - the top struct and then any reachable anonymous structs. +func typeFields(t reflect.Type) []field { + // Anonymous fields to explore at the current level and the next. + current := []field{} + next := []field{{typ: t}} + + // Count of queued names for current level and the next. + count := map[reflect.Type]int{} + nextCount := map[reflect.Type]int{} + + // Types already visited at an earlier level. + visited := map[reflect.Type]bool{} + + // Fields found. + var fields []field + + for len(next) > 0 { + current, next = next, current[:0] + count, nextCount = nextCount, map[reflect.Type]int{} + + for _, f := range current { + if visited[f.typ] { + continue + } + visited[f.typ] = true + + // Scan f.typ for fields to include. + for i := 0; i < f.typ.NumField(); i++ { + sf := f.typ.Field(i) + if sf.PkgPath != "" && !sf.Anonymous { // unexported + continue + } + opts := getOptions(sf.Tag) + if opts.skip { + continue + } + index := make([]int, len(f.index)+1) + copy(index, f.index) + index[len(f.index)] = i + + ft := sf.Type + if ft.Name() == "" && ft.Kind() == reflect.Ptr { + // Follow pointer. + ft = ft.Elem() + } + + // Record found field and index sequence. + if opts.name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { + tagged := opts.name != "" + name := opts.name + if name == "" { + name = sf.Name + } + fields = append(fields, field{name, tagged, index, ft}) + if count[f.typ] > 1 { + // If there were multiple instances, add a second, + // so that the annihilation code will see a duplicate. + // It only cares about the distinction between 1 or 2, + // so don't bother generating any more copies. + fields = append(fields, fields[len(fields)-1]) + } + continue + } + + // Record new anonymous struct to explore in next round. + nextCount[ft]++ + if nextCount[ft] == 1 { + f := field{name: ft.Name(), index: index, typ: ft} + next = append(next, f) + } + } + } + } + + sort.Sort(byName(fields)) + + // Delete all fields that are hidden by the Go rules for embedded fields, + // except that fields with TOML tags are promoted. + + // The fields are sorted in primary order of name, secondary order + // of field index length. Loop over names; for each name, delete + // hidden fields by choosing the one dominant field that survives. + out := fields[:0] + for advance, i := 0, 0; i < len(fields); i += advance { + // One iteration per name. + // Find the sequence of fields with the name of this first field. + fi := fields[i] + name := fi.name + for advance = 1; i+advance < len(fields); advance++ { + fj := fields[i+advance] + if fj.name != name { + break + } + } + if advance == 1 { // Only one field with this name + out = append(out, fi) + continue + } + dominant, ok := dominantField(fields[i : i+advance]) + if ok { + out = append(out, dominant) + } + } + + fields = out + sort.Sort(byIndex(fields)) + + return fields +} + +// dominantField looks through the fields, all of which are known to +// have the same name, to find the single field that dominates the +// others using Go's embedding rules, modified by the presence of +// TOML tags. If there are multiple top-level fields, the boolean +// will be false: This condition is an error in Go and we skip all +// the fields. +func dominantField(fields []field) (field, bool) { + // The fields are sorted in increasing index-length order. The winner + // must therefore be one with the shortest index length. Drop all + // longer entries, which is easy: just truncate the slice. + length := len(fields[0].index) + tagged := -1 // Index of first tagged field. + for i, f := range fields { + if len(f.index) > length { + fields = fields[:i] + break + } + if f.tag { + if tagged >= 0 { + // Multiple tagged fields at the same level: conflict. + // Return no field. + return field{}, false + } + tagged = i + } + } + if tagged >= 0 { + return fields[tagged], true + } + // All remaining fields have the same length. If there's more than one, + // we have a conflict (two fields named "X" at the same level) and we + // return no field. + if len(fields) > 1 { + return field{}, false + } + return fields[0], true +} + +var fieldCache struct { + sync.RWMutex + m map[reflect.Type][]field +} + +// cachedTypeFields is like typeFields but uses a cache to avoid repeated work. +func cachedTypeFields(t reflect.Type) []field { + fieldCache.RLock() + f := fieldCache.m[t] + fieldCache.RUnlock() + if f != nil { + return f + } + + // Compute fields without lock. + // Might duplicate effort but won't hold other computations back. + f = typeFields(t) + if f == nil { + f = []field{} + } + + fieldCache.Lock() + if fieldCache.m == nil { + fieldCache.m = map[reflect.Type][]field{} + } + fieldCache.m[t] = f + fieldCache.Unlock() + return f +} diff --git a/vendor/github.com/coreos/bbolt/.gitignore b/vendor/github.com/coreos/bbolt/.gitignore new file mode 100644 index 0000000..c7bd2b7 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/.gitignore @@ -0,0 +1,4 @@ +*.prof +*.test +*.swp +/bin/ diff --git a/vendor/github.com/coreos/bbolt/LICENSE b/vendor/github.com/coreos/bbolt/LICENSE new file mode 100644 index 0000000..004e77f --- /dev/null +++ b/vendor/github.com/coreos/bbolt/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013 Ben Johnson + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/coreos/bbolt/Makefile b/vendor/github.com/coreos/bbolt/Makefile new file mode 100644 index 0000000..e035e63 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/Makefile @@ -0,0 +1,18 @@ +BRANCH=`git rev-parse --abbrev-ref HEAD` +COMMIT=`git rev-parse --short HEAD` +GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)" + +default: build + +race: + @go test -v -race -test.run="TestSimulate_(100op|1000op)" + +# go get github.com/kisielk/errcheck +errcheck: + @errcheck -ignorepkg=bytes -ignore=os:Remove github.com/boltdb/bolt + +test: + @go test -v -cover . + @go test -v ./cmd/bolt + +.PHONY: fmt test diff --git a/vendor/github.com/coreos/bbolt/README.md b/vendor/github.com/coreos/bbolt/README.md new file mode 100644 index 0000000..8523e33 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/README.md @@ -0,0 +1,852 @@ +Bolt [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.2.1-green.svg) +==== + +Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas] +[LMDB project][lmdb]. The goal of the project is to provide a simple, +fast, and reliable database for projects that don't require a full database +server such as Postgres or MySQL. + +Since Bolt is meant to be used as such a low-level piece of functionality, +simplicity is key. The API will be small and only focus on getting values +and setting values. That's it. + +[hyc_symas]: https://twitter.com/hyc_symas +[lmdb]: http://symas.com/mdb/ + +## Project Status + +Bolt is stable and the API is fixed. Full unit test coverage and randomized +black box testing are used to ensure database consistency and thread safety. +Bolt is currently in high-load production environments serving databases as +large as 1TB. Many companies such as Shopify and Heroku use Bolt-backed +services every day. + +## Table of Contents + +- [Getting Started](#getting-started) + - [Installing](#installing) + - [Opening a database](#opening-a-database) + - [Transactions](#transactions) + - [Read-write transactions](#read-write-transactions) + - [Read-only transactions](#read-only-transactions) + - [Batch read-write transactions](#batch-read-write-transactions) + - [Managing transactions manually](#managing-transactions-manually) + - [Using buckets](#using-buckets) + - [Using key/value pairs](#using-keyvalue-pairs) + - [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket) + - [Iterating over keys](#iterating-over-keys) + - [Prefix scans](#prefix-scans) + - [Range scans](#range-scans) + - [ForEach()](#foreach) + - [Nested buckets](#nested-buckets) + - [Database backups](#database-backups) + - [Statistics](#statistics) + - [Read-Only Mode](#read-only-mode) + - [Mobile Use (iOS/Android)](#mobile-use-iosandroid) +- [Resources](#resources) +- [Comparison with other databases](#comparison-with-other-databases) + - [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases) + - [LevelDB, RocksDB](#leveldb-rocksdb) + - [LMDB](#lmdb) +- [Caveats & Limitations](#caveats--limitations) +- [Reading the Source](#reading-the-source) +- [Other Projects Using Bolt](#other-projects-using-bolt) + +## Getting Started + +### Installing + +To start using Bolt, install Go and run `go get`: + +```sh +$ go get github.com/boltdb/bolt/... +``` + +This will retrieve the library and install the `bolt` command line utility into +your `$GOBIN` path. + + +### Opening a database + +The top-level object in Bolt is a `DB`. It is represented as a single file on +your disk and represents a consistent snapshot of your data. + +To open your database, simply use the `bolt.Open()` function: + +```go +package main + +import ( + "log" + + "github.com/boltdb/bolt" +) + +func main() { + // Open the my.db data file in your current directory. + // It will be created if it doesn't exist. + db, err := bolt.Open("my.db", 0600, nil) + if err != nil { + log.Fatal(err) + } + defer db.Close() + + ... +} +``` + +Please note that Bolt obtains a file lock on the data file so multiple processes +cannot open the same database at the same time. Opening an already open Bolt +database will cause it to hang until the other process closes it. To prevent +an indefinite wait you can pass a timeout option to the `Open()` function: + +```go +db, err := bolt.Open("my.db", 0600, &bolt.Options{Timeout: 1 * time.Second}) +``` + + +### Transactions + +Bolt allows only one read-write transaction at a time but allows as many +read-only transactions as you want at a time. Each transaction has a consistent +view of the data as it existed when the transaction started. + +Individual transactions and all objects created from them (e.g. buckets, keys) +are not thread safe. To work with data in multiple goroutines you must start +a transaction for each one or use locking to ensure only one goroutine accesses +a transaction at a time. Creating transaction from the `DB` is thread safe. + +Read-only transactions and read-write transactions should not depend on one +another and generally shouldn't be opened simultaneously in the same goroutine. +This can cause a deadlock as the read-write transaction needs to periodically +re-map the data file but it cannot do so while a read-only transaction is open. + + +#### Read-write transactions + +To start a read-write transaction, you can use the `DB.Update()` function: + +```go +err := db.Update(func(tx *bolt.Tx) error { + ... + return nil +}) +``` + +Inside the closure, you have a consistent view of the database. You commit the +transaction by returning `nil` at the end. You can also rollback the transaction +at any point by returning an error. All database operations are allowed inside +a read-write transaction. + +Always check the return error as it will report any disk failures that can cause +your transaction to not complete. If you return an error within your closure +it will be passed through. + + +#### Read-only transactions + +To start a read-only transaction, you can use the `DB.View()` function: + +```go +err := db.View(func(tx *bolt.Tx) error { + ... + return nil +}) +``` + +You also get a consistent view of the database within this closure, however, +no mutating operations are allowed within a read-only transaction. You can only +retrieve buckets, retrieve values, and copy the database within a read-only +transaction. + + +#### Batch read-write transactions + +Each `DB.Update()` waits for disk to commit the writes. This overhead +can be minimized by combining multiple updates with the `DB.Batch()` +function: + +```go +err := db.Batch(func(tx *bolt.Tx) error { + ... + return nil +}) +``` + +Concurrent Batch calls are opportunistically combined into larger +transactions. Batch is only useful when there are multiple goroutines +calling it. + +The trade-off is that `Batch` can call the given +function multiple times, if parts of the transaction fail. The +function must be idempotent and side effects must take effect only +after a successful return from `DB.Batch()`. + +For example: don't display messages from inside the function, instead +set variables in the enclosing scope: + +```go +var id uint64 +err := db.Batch(func(tx *bolt.Tx) error { + // Find last key in bucket, decode as bigendian uint64, increment + // by one, encode back to []byte, and add new key. + ... + id = newValue + return nil +}) +if err != nil { + return ... +} +fmt.Println("Allocated ID %d", id) +``` + + +#### Managing transactions manually + +The `DB.View()` and `DB.Update()` functions are wrappers around the `DB.Begin()` +function. These helper functions will start the transaction, execute a function, +and then safely close your transaction if an error is returned. This is the +recommended way to use Bolt transactions. + +However, sometimes you may want to manually start and end your transactions. +You can use the `Tx.Begin()` function directly but **please** be sure to close +the transaction. + +```go +// Start a writable transaction. +tx, err := db.Begin(true) +if err != nil { + return err +} +defer tx.Rollback() + +// Use the transaction... +_, err := tx.CreateBucket([]byte("MyBucket")) +if err != nil { + return err +} + +// Commit the transaction and check for error. +if err := tx.Commit(); err != nil { + return err +} +``` + +The first argument to `DB.Begin()` is a boolean stating if the transaction +should be writable. + + +### Using buckets + +Buckets are collections of key/value pairs within the database. All keys in a +bucket must be unique. You can create a bucket using the `DB.CreateBucket()` +function: + +```go +db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("MyBucket")) + if err != nil { + return fmt.Errorf("create bucket: %s", err) + } + return nil +}) +``` + +You can also create a bucket only if it doesn't exist by using the +`Tx.CreateBucketIfNotExists()` function. It's a common pattern to call this +function for all your top-level buckets after you open your database so you can +guarantee that they exist for future transactions. + +To delete a bucket, simply call the `Tx.DeleteBucket()` function. + + +### Using key/value pairs + +To save a key/value pair to a bucket, use the `Bucket.Put()` function: + +```go +db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("MyBucket")) + err := b.Put([]byte("answer"), []byte("42")) + return err +}) +``` + +This will set the value of the `"answer"` key to `"42"` in the `MyBucket` +bucket. To retrieve this value, we can use the `Bucket.Get()` function: + +```go +db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("MyBucket")) + v := b.Get([]byte("answer")) + fmt.Printf("The answer is: %s\n", v) + return nil +}) +``` + +The `Get()` function does not return an error because its operation is +guaranteed to work (unless there is some kind of system failure). If the key +exists then it will return its byte slice value. If it doesn't exist then it +will return `nil`. It's important to note that you can have a zero-length value +set to a key which is different than the key not existing. + +Use the `Bucket.Delete()` function to delete a key from the bucket. + +Please note that values returned from `Get()` are only valid while the +transaction is open. If you need to use a value outside of the transaction +then you must use `copy()` to copy it to another byte slice. + + +### Autoincrementing integer for the bucket +By using the `NextSequence()` function, you can let Bolt determine a sequence +which can be used as the unique identifier for your key/value pairs. See the +example below. + +```go +// CreateUser saves u to the store. The new user ID is set on u once the data is persisted. +func (s *Store) CreateUser(u *User) error { + return s.db.Update(func(tx *bolt.Tx) error { + // Retrieve the users bucket. + // This should be created when the DB is first opened. + b := tx.Bucket([]byte("users")) + + // Generate ID for the user. + // This returns an error only if the Tx is closed or not writeable. + // That can't happen in an Update() call so I ignore the error check. + id, _ := b.NextSequence() + u.ID = int(id) + + // Marshal user data into bytes. + buf, err := json.Marshal(u) + if err != nil { + return err + } + + // Persist bytes to users bucket. + return b.Put(itob(u.ID), buf) + }) +} + +// itob returns an 8-byte big endian representation of v. +func itob(v int) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(v)) + return b +} + +type User struct { + ID int + ... +} +``` + +### Iterating over keys + +Bolt stores its keys in byte-sorted order within a bucket. This makes sequential +iteration over these keys extremely fast. To iterate over keys we'll use a +`Cursor`: + +```go +db.View(func(tx *bolt.Tx) error { + // Assume bucket exists and has keys + b := tx.Bucket([]byte("MyBucket")) + + c := b.Cursor() + + for k, v := c.First(); k != nil; k, v = c.Next() { + fmt.Printf("key=%s, value=%s\n", k, v) + } + + return nil +}) +``` + +The cursor allows you to move to a specific point in the list of keys and move +forward or backward through the keys one at a time. + +The following functions are available on the cursor: + +``` +First() Move to the first key. +Last() Move to the last key. +Seek() Move to a specific key. +Next() Move to the next key. +Prev() Move to the previous key. +``` + +Each of those functions has a return signature of `(key []byte, value []byte)`. +When you have iterated to the end of the cursor then `Next()` will return a +`nil` key. You must seek to a position using `First()`, `Last()`, or `Seek()` +before calling `Next()` or `Prev()`. If you do not seek to a position then +these functions will return a `nil` key. + +During iteration, if the key is non-`nil` but the value is `nil`, that means +the key refers to a bucket rather than a value. Use `Bucket.Bucket()` to +access the sub-bucket. + + +#### Prefix scans + +To iterate over a key prefix, you can combine `Seek()` and `bytes.HasPrefix()`: + +```go +db.View(func(tx *bolt.Tx) error { + // Assume bucket exists and has keys + c := tx.Bucket([]byte("MyBucket")).Cursor() + + prefix := []byte("1234") + for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() { + fmt.Printf("key=%s, value=%s\n", k, v) + } + + return nil +}) +``` + +#### Range scans + +Another common use case is scanning over a range such as a time range. If you +use a sortable time encoding such as RFC3339 then you can query a specific +date range like this: + +```go +db.View(func(tx *bolt.Tx) error { + // Assume our events bucket exists and has RFC3339 encoded time keys. + c := tx.Bucket([]byte("Events")).Cursor() + + // Our time range spans the 90's decade. + min := []byte("1990-01-01T00:00:00Z") + max := []byte("2000-01-01T00:00:00Z") + + // Iterate over the 90's. + for k, v := c.Seek(min); k != nil && bytes.Compare(k, max) <= 0; k, v = c.Next() { + fmt.Printf("%s: %s\n", k, v) + } + + return nil +}) +``` + +Note that, while RFC3339 is sortable, the Golang implementation of RFC3339Nano does not use a fixed number of digits after the decimal point and is therefore not sortable. + + +#### ForEach() + +You can also use the function `ForEach()` if you know you'll be iterating over +all the keys in a bucket: + +```go +db.View(func(tx *bolt.Tx) error { + // Assume bucket exists and has keys + b := tx.Bucket([]byte("MyBucket")) + + b.ForEach(func(k, v []byte) error { + fmt.Printf("key=%s, value=%s\n", k, v) + return nil + }) + return nil +}) +``` + + +### Nested buckets + +You can also store a bucket in a key to create nested buckets. The API is the +same as the bucket management API on the `DB` object: + +```go +func (*Bucket) CreateBucket(key []byte) (*Bucket, error) +func (*Bucket) CreateBucketIfNotExists(key []byte) (*Bucket, error) +func (*Bucket) DeleteBucket(key []byte) error +``` + + +### Database backups + +Bolt is a single file so it's easy to backup. You can use the `Tx.WriteTo()` +function to write a consistent view of the database to a writer. If you call +this from a read-only transaction, it will perform a hot backup and not block +your other database reads and writes. + +By default, it will use a regular file handle which will utilize the operating +system's page cache. See the [`Tx`](https://godoc.org/github.com/boltdb/bolt#Tx) +documentation for information about optimizing for larger-than-RAM datasets. + +One common use case is to backup over HTTP so you can use tools like `cURL` to +do database backups: + +```go +func BackupHandleFunc(w http.ResponseWriter, req *http.Request) { + err := db.View(func(tx *bolt.Tx) error { + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", `attachment; filename="my.db"`) + w.Header().Set("Content-Length", strconv.Itoa(int(tx.Size()))) + _, err := tx.WriteTo(w) + return err + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} +``` + +Then you can backup using this command: + +```sh +$ curl http://localhost/backup > my.db +``` + +Or you can open your browser to `http://localhost/backup` and it will download +automatically. + +If you want to backup to another file you can use the `Tx.CopyFile()` helper +function. + + +### Statistics + +The database keeps a running count of many of the internal operations it +performs so you can better understand what's going on. By grabbing a snapshot +of these stats at two points in time we can see what operations were performed +in that time range. + +For example, we could start a goroutine to log stats every 10 seconds: + +```go +go func() { + // Grab the initial stats. + prev := db.Stats() + + for { + // Wait for 10s. + time.Sleep(10 * time.Second) + + // Grab the current stats and diff them. + stats := db.Stats() + diff := stats.Sub(&prev) + + // Encode stats to JSON and print to STDERR. + json.NewEncoder(os.Stderr).Encode(diff) + + // Save stats for the next loop. + prev = stats + } +}() +``` + +It's also useful to pipe these stats to a service such as statsd for monitoring +or to provide an HTTP endpoint that will perform a fixed-length sample. + + +### Read-Only Mode + +Sometimes it is useful to create a shared, read-only Bolt database. To this, +set the `Options.ReadOnly` flag when opening your database. Read-only mode +uses a shared lock to allow multiple processes to read from the database but +it will block any processes from opening the database in read-write mode. + +```go +db, err := bolt.Open("my.db", 0666, &bolt.Options{ReadOnly: true}) +if err != nil { + log.Fatal(err) +} +``` + +### Mobile Use (iOS/Android) + +Bolt is able to run on mobile devices by leveraging the binding feature of the +[gomobile](https://github.com/golang/mobile) tool. Create a struct that will +contain your database logic and a reference to a `*bolt.DB` with a initializing +constructor that takes in a filepath where the database file will be stored. +Neither Android nor iOS require extra permissions or cleanup from using this method. + +```go +func NewBoltDB(filepath string) *BoltDB { + db, err := bolt.Open(filepath+"/demo.db", 0600, nil) + if err != nil { + log.Fatal(err) + } + + return &BoltDB{db} +} + +type BoltDB struct { + db *bolt.DB + ... +} + +func (b *BoltDB) Path() string { + return b.db.Path() +} + +func (b *BoltDB) Close() { + b.db.Close() +} +``` + +Database logic should be defined as methods on this wrapper struct. + +To initialize this struct from the native language (both platforms now sync +their local storage to the cloud. These snippets disable that functionality for the +database file): + +#### Android + +```java +String path; +if (android.os.Build.VERSION.SDK_INT >=android.os.Build.VERSION_CODES.LOLLIPOP){ + path = getNoBackupFilesDir().getAbsolutePath(); +} else{ + path = getFilesDir().getAbsolutePath(); +} +Boltmobiledemo.BoltDB boltDB = Boltmobiledemo.NewBoltDB(path) +``` + +#### iOS + +```objc +- (void)demo { + NSString* path = [NSSearchPathForDirectoriesInDomains(NSLibraryDirectory, + NSUserDomainMask, + YES) objectAtIndex:0]; + GoBoltmobiledemoBoltDB * demo = GoBoltmobiledemoNewBoltDB(path); + [self addSkipBackupAttributeToItemAtPath:demo.path]; + //Some DB Logic would go here + [demo close]; +} + +- (BOOL)addSkipBackupAttributeToItemAtPath:(NSString *) filePathString +{ + NSURL* URL= [NSURL fileURLWithPath: filePathString]; + assert([[NSFileManager defaultManager] fileExistsAtPath: [URL path]]); + + NSError *error = nil; + BOOL success = [URL setResourceValue: [NSNumber numberWithBool: YES] + forKey: NSURLIsExcludedFromBackupKey error: &error]; + if(!success){ + NSLog(@"Error excluding %@ from backup %@", [URL lastPathComponent], error); + } + return success; +} + +``` + +## Resources + +For more information on getting started with Bolt, check out the following articles: + +* [Intro to BoltDB: Painless Performant Persistence](http://npf.io/2014/07/intro-to-boltdb-painless-performant-persistence/) by [Nate Finch](https://github.com/natefinch). +* [Bolt -- an embedded key/value database for Go](https://www.progville.com/go/bolt-embedded-db-golang/) by Progville + + +## Comparison with other databases + +### Postgres, MySQL, & other relational databases + +Relational databases structure data into rows and are only accessible through +the use of SQL. This approach provides flexibility in how you store and query +your data but also incurs overhead in parsing and planning SQL statements. Bolt +accesses all data by a byte slice key. This makes Bolt fast to read and write +data by key but provides no built-in support for joining values together. + +Most relational databases (with the exception of SQLite) are standalone servers +that run separately from your application. This gives your systems +flexibility to connect multiple application servers to a single database +server but also adds overhead in serializing and transporting data over the +network. Bolt runs as a library included in your application so all data access +has to go through your application's process. This brings data closer to your +application but limits multi-process access to the data. + + +### LevelDB, RocksDB + +LevelDB and its derivatives (RocksDB, HyperLevelDB) are similar to Bolt in that +they are libraries bundled into the application, however, their underlying +structure is a log-structured merge-tree (LSM tree). An LSM tree optimizes +random writes by using a write ahead log and multi-tiered, sorted files called +SSTables. Bolt uses a B+tree internally and only a single file. Both approaches +have trade-offs. + +If you require a high random write throughput (>10,000 w/sec) or you need to use +spinning disks then LevelDB could be a good choice. If your application is +read-heavy or does a lot of range scans then Bolt could be a good choice. + +One other important consideration is that LevelDB does not have transactions. +It supports batch writing of key/values pairs and it supports read snapshots +but it will not give you the ability to do a compare-and-swap operation safely. +Bolt supports fully serializable ACID transactions. + + +### LMDB + +Bolt was originally a port of LMDB so it is architecturally similar. Both use +a B+tree, have ACID semantics with fully serializable transactions, and support +lock-free MVCC using a single writer and multiple readers. + +The two projects have somewhat diverged. LMDB heavily focuses on raw performance +while Bolt has focused on simplicity and ease of use. For example, LMDB allows +several unsafe actions such as direct writes for the sake of performance. Bolt +opts to disallow actions which can leave the database in a corrupted state. The +only exception to this in Bolt is `DB.NoSync`. + +There are also a few differences in API. LMDB requires a maximum mmap size when +opening an `mdb_env` whereas Bolt will handle incremental mmap resizing +automatically. LMDB overloads the getter and setter functions with multiple +flags whereas Bolt splits these specialized cases into their own functions. + + +## Caveats & Limitations + +It's important to pick the right tool for the job and Bolt is no exception. +Here are a few things to note when evaluating and using Bolt: + +* Bolt is good for read intensive workloads. Sequential write performance is + also fast but random writes can be slow. You can use `DB.Batch()` or add a + write-ahead log to help mitigate this issue. + +* Bolt uses a B+tree internally so there can be a lot of random page access. + SSDs provide a significant performance boost over spinning disks. + +* Try to avoid long running read transactions. Bolt uses copy-on-write so + old pages cannot be reclaimed while an old transaction is using them. + +* Byte slices returned from Bolt are only valid during a transaction. Once the + transaction has been committed or rolled back then the memory they point to + can be reused by a new page or can be unmapped from virtual memory and you'll + see an `unexpected fault address` panic when accessing it. + +* Be careful when using `Bucket.FillPercent`. Setting a high fill percent for + buckets that have random inserts will cause your database to have very poor + page utilization. + +* Use larger buckets in general. Smaller buckets causes poor page utilization + once they become larger than the page size (typically 4KB). + +* Bulk loading a lot of random writes into a new bucket can be slow as the + page will not split until the transaction is committed. Randomly inserting + more than 100,000 key/value pairs into a single new bucket in a single + transaction is not advised. + +* Bolt uses a memory-mapped file so the underlying operating system handles the + caching of the data. Typically, the OS will cache as much of the file as it + can in memory and will release memory as needed to other processes. This means + that Bolt can show very high memory usage when working with large databases. + However, this is expected and the OS will release memory as needed. Bolt can + handle databases much larger than the available physical RAM, provided its + memory-map fits in the process virtual address space. It may be problematic + on 32-bits systems. + +* The data structures in the Bolt database are memory mapped so the data file + will be endian specific. This means that you cannot copy a Bolt file from a + little endian machine to a big endian machine and have it work. For most + users this is not a concern since most modern CPUs are little endian. + +* Because of the way pages are laid out on disk, Bolt cannot truncate data files + and return free pages back to the disk. Instead, Bolt maintains a free list + of unused pages within its data file. These free pages can be reused by later + transactions. This works well for many use cases as databases generally tend + to grow. However, it's important to note that deleting large chunks of data + will not allow you to reclaim that space on disk. + + For more information on page allocation, [see this comment][page-allocation]. + +[page-allocation]: https://github.com/boltdb/bolt/issues/308#issuecomment-74811638 + + +## Reading the Source + +Bolt is a relatively small code base (<3KLOC) for an embedded, serializable, +transactional key/value database so it can be a good starting point for people +interested in how databases work. + +The best places to start are the main entry points into Bolt: + +- `Open()` - Initializes the reference to the database. It's responsible for + creating the database if it doesn't exist, obtaining an exclusive lock on the + file, reading the meta pages, & memory-mapping the file. + +- `DB.Begin()` - Starts a read-only or read-write transaction depending on the + value of the `writable` argument. This requires briefly obtaining the "meta" + lock to keep track of open transactions. Only one read-write transaction can + exist at a time so the "rwlock" is acquired during the life of a read-write + transaction. + +- `Bucket.Put()` - Writes a key/value pair into a bucket. After validating the + arguments, a cursor is used to traverse the B+tree to the page and position + where they key & value will be written. Once the position is found, the bucket + materializes the underlying page and the page's parent pages into memory as + "nodes". These nodes are where mutations occur during read-write transactions. + These changes get flushed to disk during commit. + +- `Bucket.Get()` - Retrieves a key/value pair from a bucket. This uses a cursor + to move to the page & position of a key/value pair. During a read-only + transaction, the key and value data is returned as a direct reference to the + underlying mmap file so there's no allocation overhead. For read-write + transactions, this data may reference the mmap file or one of the in-memory + node values. + +- `Cursor` - This object is simply for traversing the B+tree of on-disk pages + or in-memory nodes. It can seek to a specific key, move to the first or last + value, or it can move forward or backward. The cursor handles the movement up + and down the B+tree transparently to the end user. + +- `Tx.Commit()` - Converts the in-memory dirty nodes and the list of free pages + into pages to be written to disk. Writing to disk then occurs in two phases. + First, the dirty pages are written to disk and an `fsync()` occurs. Second, a + new meta page with an incremented transaction ID is written and another + `fsync()` occurs. This two phase write ensures that partially written data + pages are ignored in the event of a crash since the meta page pointing to them + is never written. Partially written meta pages are invalidated because they + are written with a checksum. + +If you have additional notes that could be helpful for others, please submit +them via pull request. + + +## Other Projects Using Bolt + +Below is a list of public, open source projects that use Bolt: + +* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files. +* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard. +* [Bazil](https://bazil.org/) - A file system that lets your data reside where it is most convenient for it to reside. +* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb. +* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics. +* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects. +* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday. +* [ChainStore](https://github.com/pressly/chainstore) - Simple key-value interface to a variety of storage engines organized as a chain of operations. +* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite. +* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin". +* [event-shuttle](https://github.com/sclasen/event-shuttle) - A Unix system service to collect and reliably deliver messages to Kafka. +* [ipxed](https://github.com/kelseyhightower/ipxed) - Web interface and api for ipxed. +* [BoltStore](https://github.com/yosssi/boltstore) - Session store using Bolt. +* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site. +* [LedisDB](https://github.com/siddontang/ledisdb) - A high performance NoSQL, using Bolt as optional storage. +* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters. +* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend. +* [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend. +* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server. +* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read. +* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics. +* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data. +* [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system. +* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware. +* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs. +* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems. +* [stow](https://github.com/djherbis/stow) - a persistence manager for objects + backed by boltdb. +* [buckets](https://github.com/joyrexus/buckets) - a bolt wrapper streamlining + simple tx and key scans. +* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets. +* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service +* [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service. +* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners. +* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores. +* [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB. +* [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB. +* [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings. +* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend. +* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files. +* [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter. + +If you are using Bolt in a project please send a pull request to add it to the list. diff --git a/vendor/github.com/coreos/bbolt/appveyor.yml b/vendor/github.com/coreos/bbolt/appveyor.yml new file mode 100644 index 0000000..6e26e94 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/appveyor.yml @@ -0,0 +1,18 @@ +version: "{build}" + +os: Windows Server 2012 R2 + +clone_folder: c:\gopath\src\github.com\boltdb\bolt + +environment: + GOPATH: c:\gopath + +install: + - echo %PATH% + - echo %GOPATH% + - go version + - go env + - go get -v -t ./... + +build_script: + - go test -v ./... diff --git a/vendor/github.com/coreos/bbolt/bolt_386.go b/vendor/github.com/coreos/bbolt/bolt_386.go new file mode 100644 index 0000000..e659bfb --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bolt_386.go @@ -0,0 +1,7 @@ +package bolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0x7FFFFFFF // 2GB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0xFFFFFFF diff --git a/vendor/github.com/coreos/bbolt/bolt_amd64.go b/vendor/github.com/coreos/bbolt/bolt_amd64.go new file mode 100644 index 0000000..cca6b7e --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bolt_amd64.go @@ -0,0 +1,7 @@ +package bolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0xFFFFFFFFFFFF // 256TB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0x7FFFFFFF diff --git a/vendor/github.com/coreos/bbolt/bolt_arm.go b/vendor/github.com/coreos/bbolt/bolt_arm.go new file mode 100644 index 0000000..e659bfb --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bolt_arm.go @@ -0,0 +1,7 @@ +package bolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0x7FFFFFFF // 2GB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0xFFFFFFF diff --git a/vendor/github.com/coreos/bbolt/bolt_arm64.go b/vendor/github.com/coreos/bbolt/bolt_arm64.go new file mode 100644 index 0000000..6d23093 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bolt_arm64.go @@ -0,0 +1,9 @@ +// +build arm64 + +package bolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0xFFFFFFFFFFFF // 256TB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0x7FFFFFFF diff --git a/vendor/github.com/coreos/bbolt/bolt_linux.go b/vendor/github.com/coreos/bbolt/bolt_linux.go new file mode 100644 index 0000000..2b67666 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bolt_linux.go @@ -0,0 +1,10 @@ +package bolt + +import ( + "syscall" +) + +// fdatasync flushes written data to a file descriptor. +func fdatasync(db *DB) error { + return syscall.Fdatasync(int(db.file.Fd())) +} diff --git a/vendor/github.com/coreos/bbolt/bolt_openbsd.go b/vendor/github.com/coreos/bbolt/bolt_openbsd.go new file mode 100644 index 0000000..7058c3d --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bolt_openbsd.go @@ -0,0 +1,27 @@ +package bolt + +import ( + "syscall" + "unsafe" +) + +const ( + msAsync = 1 << iota // perform asynchronous writes + msSync // perform synchronous writes + msInvalidate // invalidate cached data +) + +func msync(db *DB) error { + _, _, errno := syscall.Syscall(syscall.SYS_MSYNC, uintptr(unsafe.Pointer(db.data)), uintptr(db.datasz), msInvalidate) + if errno != 0 { + return errno + } + return nil +} + +func fdatasync(db *DB) error { + if db.data != nil { + return msync(db) + } + return db.file.Sync() +} diff --git a/vendor/github.com/coreos/bbolt/bolt_ppc.go b/vendor/github.com/coreos/bbolt/bolt_ppc.go new file mode 100644 index 0000000..645ddc3 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bolt_ppc.go @@ -0,0 +1,9 @@ +// +build ppc + +package bolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0x7FFFFFFF // 2GB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0xFFFFFFF diff --git a/vendor/github.com/coreos/bbolt/bolt_ppc64.go b/vendor/github.com/coreos/bbolt/bolt_ppc64.go new file mode 100644 index 0000000..2dc6be0 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bolt_ppc64.go @@ -0,0 +1,9 @@ +// +build ppc64 + +package bolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0xFFFFFFFFFFFF // 256TB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0x7FFFFFFF diff --git a/vendor/github.com/coreos/bbolt/bolt_ppc64le.go b/vendor/github.com/coreos/bbolt/bolt_ppc64le.go new file mode 100644 index 0000000..8351e12 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bolt_ppc64le.go @@ -0,0 +1,9 @@ +// +build ppc64le + +package bolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0xFFFFFFFFFFFF // 256TB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0x7FFFFFFF diff --git a/vendor/github.com/coreos/bbolt/bolt_s390x.go b/vendor/github.com/coreos/bbolt/bolt_s390x.go new file mode 100644 index 0000000..f4dd26b --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bolt_s390x.go @@ -0,0 +1,9 @@ +// +build s390x + +package bolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0xFFFFFFFFFFFF // 256TB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0x7FFFFFFF diff --git a/vendor/github.com/coreos/bbolt/bolt_unix.go b/vendor/github.com/coreos/bbolt/bolt_unix.go new file mode 100644 index 0000000..cad62dd --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bolt_unix.go @@ -0,0 +1,89 @@ +// +build !windows,!plan9,!solaris + +package bolt + +import ( + "fmt" + "os" + "syscall" + "time" + "unsafe" +) + +// flock acquires an advisory lock on a file descriptor. +func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { + var t time.Time + for { + // If we're beyond our timeout then return an error. + // This can only occur after we've attempted a flock once. + if t.IsZero() { + t = time.Now() + } else if timeout > 0 && time.Since(t) > timeout { + return ErrTimeout + } + flag := syscall.LOCK_SH + if exclusive { + flag = syscall.LOCK_EX + } + + // Otherwise attempt to obtain an exclusive lock. + err := syscall.Flock(int(db.file.Fd()), flag|syscall.LOCK_NB) + if err == nil { + return nil + } else if err != syscall.EWOULDBLOCK { + return err + } + + // Wait for a bit and try again. + time.Sleep(50 * time.Millisecond) + } +} + +// funlock releases an advisory lock on a file descriptor. +func funlock(db *DB) error { + return syscall.Flock(int(db.file.Fd()), syscall.LOCK_UN) +} + +// mmap memory maps a DB's data file. +func mmap(db *DB, sz int) error { + // Map the data file to memory. + b, err := syscall.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED|db.MmapFlags) + if err != nil { + return err + } + + // Advise the kernel that the mmap is accessed randomly. + if err := madvise(b, syscall.MADV_RANDOM); err != nil { + return fmt.Errorf("madvise: %s", err) + } + + // Save the original byte slice and convert to a byte array pointer. + db.dataref = b + db.data = (*[maxMapSize]byte)(unsafe.Pointer(&b[0])) + db.datasz = sz + return nil +} + +// munmap unmaps a DB's data file from memory. +func munmap(db *DB) error { + // Ignore the unmap if we have no mapped data. + if db.dataref == nil { + return nil + } + + // Unmap using the original byte slice. + err := syscall.Munmap(db.dataref) + db.dataref = nil + db.data = nil + db.datasz = 0 + return err +} + +// NOTE: This function is copied from stdlib because it is not available on darwin. +func madvise(b []byte, advice int) (err error) { + _, _, e1 := syscall.Syscall(syscall.SYS_MADVISE, uintptr(unsafe.Pointer(&b[0])), uintptr(len(b)), uintptr(advice)) + if e1 != 0 { + err = e1 + } + return +} diff --git a/vendor/github.com/coreos/bbolt/bolt_unix_solaris.go b/vendor/github.com/coreos/bbolt/bolt_unix_solaris.go new file mode 100644 index 0000000..307bf2b --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bolt_unix_solaris.go @@ -0,0 +1,90 @@ +package bolt + +import ( + "fmt" + "os" + "syscall" + "time" + "unsafe" + + "golang.org/x/sys/unix" +) + +// flock acquires an advisory lock on a file descriptor. +func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { + var t time.Time + for { + // If we're beyond our timeout then return an error. + // This can only occur after we've attempted a flock once. + if t.IsZero() { + t = time.Now() + } else if timeout > 0 && time.Since(t) > timeout { + return ErrTimeout + } + var lock syscall.Flock_t + lock.Start = 0 + lock.Len = 0 + lock.Pid = 0 + lock.Whence = 0 + lock.Pid = 0 + if exclusive { + lock.Type = syscall.F_WRLCK + } else { + lock.Type = syscall.F_RDLCK + } + err := syscall.FcntlFlock(db.file.Fd(), syscall.F_SETLK, &lock) + if err == nil { + return nil + } else if err != syscall.EAGAIN { + return err + } + + // Wait for a bit and try again. + time.Sleep(50 * time.Millisecond) + } +} + +// funlock releases an advisory lock on a file descriptor. +func funlock(db *DB) error { + var lock syscall.Flock_t + lock.Start = 0 + lock.Len = 0 + lock.Type = syscall.F_UNLCK + lock.Whence = 0 + return syscall.FcntlFlock(uintptr(db.file.Fd()), syscall.F_SETLK, &lock) +} + +// mmap memory maps a DB's data file. +func mmap(db *DB, sz int) error { + // Map the data file to memory. + b, err := unix.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED|db.MmapFlags) + if err != nil { + return err + } + + // Advise the kernel that the mmap is accessed randomly. + if err := unix.Madvise(b, syscall.MADV_RANDOM); err != nil { + return fmt.Errorf("madvise: %s", err) + } + + // Save the original byte slice and convert to a byte array pointer. + db.dataref = b + db.data = (*[maxMapSize]byte)(unsafe.Pointer(&b[0])) + db.datasz = sz + return nil +} + +// munmap unmaps a DB's data file from memory. +func munmap(db *DB) error { + // Ignore the unmap if we have no mapped data. + if db.dataref == nil { + return nil + } + + // Unmap using the original byte slice. + err := unix.Munmap(db.dataref) + db.dataref = nil + db.data = nil + db.datasz = 0 + return err +} diff --git a/vendor/github.com/coreos/bbolt/bolt_windows.go b/vendor/github.com/coreos/bbolt/bolt_windows.go new file mode 100644 index 0000000..d538e6a --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bolt_windows.go @@ -0,0 +1,144 @@ +package bolt + +import ( + "fmt" + "os" + "syscall" + "time" + "unsafe" +) + +// LockFileEx code derived from golang build filemutex_windows.go @ v1.5.1 +var ( + modkernel32 = syscall.NewLazyDLL("kernel32.dll") + procLockFileEx = modkernel32.NewProc("LockFileEx") + procUnlockFileEx = modkernel32.NewProc("UnlockFileEx") +) + +const ( + lockExt = ".lock" + + // see https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx + flagLockExclusive = 2 + flagLockFailImmediately = 1 + + // see https://msdn.microsoft.com/en-us/library/windows/desktop/ms681382(v=vs.85).aspx + errLockViolation syscall.Errno = 0x21 +) + +func lockFileEx(h syscall.Handle, flags, reserved, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) { + r, _, err := procLockFileEx.Call(uintptr(h), uintptr(flags), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol))) + if r == 0 { + return err + } + return nil +} + +func unlockFileEx(h syscall.Handle, reserved, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) { + r, _, err := procUnlockFileEx.Call(uintptr(h), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol)), 0) + if r == 0 { + return err + } + return nil +} + +// fdatasync flushes written data to a file descriptor. +func fdatasync(db *DB) error { + return db.file.Sync() +} + +// flock acquires an advisory lock on a file descriptor. +func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { + // Create a separate lock file on windows because a process + // cannot share an exclusive lock on the same file. This is + // needed during Tx.WriteTo(). + f, err := os.OpenFile(db.path+lockExt, os.O_CREATE, mode) + if err != nil { + return err + } + db.lockfile = f + + var t time.Time + for { + // If we're beyond our timeout then return an error. + // This can only occur after we've attempted a flock once. + if t.IsZero() { + t = time.Now() + } else if timeout > 0 && time.Since(t) > timeout { + return ErrTimeout + } + + var flag uint32 = flagLockFailImmediately + if exclusive { + flag |= flagLockExclusive + } + + err := lockFileEx(syscall.Handle(db.lockfile.Fd()), flag, 0, 1, 0, &syscall.Overlapped{}) + if err == nil { + return nil + } else if err != errLockViolation { + return err + } + + // Wait for a bit and try again. + time.Sleep(50 * time.Millisecond) + } +} + +// funlock releases an advisory lock on a file descriptor. +func funlock(db *DB) error { + err := unlockFileEx(syscall.Handle(db.lockfile.Fd()), 0, 1, 0, &syscall.Overlapped{}) + db.lockfile.Close() + os.Remove(db.path+lockExt) + return err +} + +// mmap memory maps a DB's data file. +// Based on: https://github.com/edsrzf/mmap-go +func mmap(db *DB, sz int) error { + if !db.readOnly { + // Truncate the database to the size of the mmap. + if err := db.file.Truncate(int64(sz)); err != nil { + return fmt.Errorf("truncate: %s", err) + } + } + + // Open a file mapping handle. + sizelo := uint32(sz >> 32) + sizehi := uint32(sz) & 0xffffffff + h, errno := syscall.CreateFileMapping(syscall.Handle(db.file.Fd()), nil, syscall.PAGE_READONLY, sizelo, sizehi, nil) + if h == 0 { + return os.NewSyscallError("CreateFileMapping", errno) + } + + // Create the memory map. + addr, errno := syscall.MapViewOfFile(h, syscall.FILE_MAP_READ, 0, 0, uintptr(sz)) + if addr == 0 { + return os.NewSyscallError("MapViewOfFile", errno) + } + + // Close mapping handle. + if err := syscall.CloseHandle(syscall.Handle(h)); err != nil { + return os.NewSyscallError("CloseHandle", err) + } + + // Convert to a byte array. + db.data = ((*[maxMapSize]byte)(unsafe.Pointer(addr))) + db.datasz = sz + + return nil +} + +// munmap unmaps a pointer from a file. +// Based on: https://github.com/edsrzf/mmap-go +func munmap(db *DB) error { + if db.data == nil { + return nil + } + + addr := (uintptr)(unsafe.Pointer(&db.data[0])) + if err := syscall.UnmapViewOfFile(addr); err != nil { + return os.NewSyscallError("UnmapViewOfFile", err) + } + return nil +} diff --git a/vendor/github.com/coreos/bbolt/boltsync_unix.go b/vendor/github.com/coreos/bbolt/boltsync_unix.go new file mode 100644 index 0000000..f504425 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/boltsync_unix.go @@ -0,0 +1,8 @@ +// +build !windows,!plan9,!linux,!openbsd + +package bolt + +// fdatasync flushes written data to a file descriptor. +func fdatasync(db *DB) error { + return db.file.Sync() +} diff --git a/vendor/github.com/coreos/bbolt/bucket.go b/vendor/github.com/coreos/bbolt/bucket.go new file mode 100644 index 0000000..d2f8c52 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bucket.go @@ -0,0 +1,748 @@ +package bolt + +import ( + "bytes" + "fmt" + "unsafe" +) + +const ( + // MaxKeySize is the maximum length of a key, in bytes. + MaxKeySize = 32768 + + // MaxValueSize is the maximum length of a value, in bytes. + MaxValueSize = (1 << 31) - 2 +) + +const ( + maxUint = ^uint(0) + minUint = 0 + maxInt = int(^uint(0) >> 1) + minInt = -maxInt - 1 +) + +const bucketHeaderSize = int(unsafe.Sizeof(bucket{})) + +const ( + minFillPercent = 0.1 + maxFillPercent = 1.0 +) + +// DefaultFillPercent is the percentage that split pages are filled. +// This value can be changed by setting Bucket.FillPercent. +const DefaultFillPercent = 0.5 + +// Bucket represents a collection of key/value pairs inside the database. +type Bucket struct { + *bucket + tx *Tx // the associated transaction + buckets map[string]*Bucket // subbucket cache + page *page // inline page reference + rootNode *node // materialized node for the root page. + nodes map[pgid]*node // node cache + + // Sets the threshold for filling nodes when they split. By default, + // the bucket will fill to 50% but it can be useful to increase this + // amount if you know that your write workloads are mostly append-only. + // + // This is non-persisted across transactions so it must be set in every Tx. + FillPercent float64 +} + +// bucket represents the on-file representation of a bucket. +// This is stored as the "value" of a bucket key. If the bucket is small enough, +// then its root page can be stored inline in the "value", after the bucket +// header. In the case of inline buckets, the "root" will be 0. +type bucket struct { + root pgid // page id of the bucket's root-level page + sequence uint64 // monotonically incrementing, used by NextSequence() +} + +// newBucket returns a new bucket associated with a transaction. +func newBucket(tx *Tx) Bucket { + var b = Bucket{tx: tx, FillPercent: DefaultFillPercent} + if tx.writable { + b.buckets = make(map[string]*Bucket) + b.nodes = make(map[pgid]*node) + } + return b +} + +// Tx returns the tx of the bucket. +func (b *Bucket) Tx() *Tx { + return b.tx +} + +// Root returns the root of the bucket. +func (b *Bucket) Root() pgid { + return b.root +} + +// Writable returns whether the bucket is writable. +func (b *Bucket) Writable() bool { + return b.tx.writable +} + +// Cursor creates a cursor associated with the bucket. +// The cursor is only valid as long as the transaction is open. +// Do not use a cursor after the transaction is closed. +func (b *Bucket) Cursor() *Cursor { + // Update transaction statistics. + b.tx.stats.CursorCount++ + + // Allocate and return a cursor. + return &Cursor{ + bucket: b, + stack: make([]elemRef, 0), + } +} + +// Bucket retrieves a nested bucket by name. +// Returns nil if the bucket does not exist. +// The bucket instance is only valid for the lifetime of the transaction. +func (b *Bucket) Bucket(name []byte) *Bucket { + if b.buckets != nil { + if child := b.buckets[string(name)]; child != nil { + return child + } + } + + // Move cursor to key. + c := b.Cursor() + k, v, flags := c.seek(name) + + // Return nil if the key doesn't exist or it is not a bucket. + if !bytes.Equal(name, k) || (flags&bucketLeafFlag) == 0 { + return nil + } + + // Otherwise create a bucket and cache it. + var child = b.openBucket(v) + if b.buckets != nil { + b.buckets[string(name)] = child + } + + return child +} + +// Helper method that re-interprets a sub-bucket value +// from a parent into a Bucket +func (b *Bucket) openBucket(value []byte) *Bucket { + var child = newBucket(b.tx) + + // If this is a writable transaction then we need to copy the bucket entry. + // Read-only transactions can point directly at the mmap entry. + if b.tx.writable { + child.bucket = &bucket{} + *child.bucket = *(*bucket)(unsafe.Pointer(&value[0])) + } else { + child.bucket = (*bucket)(unsafe.Pointer(&value[0])) + } + + // Save a reference to the inline page if the bucket is inline. + if child.root == 0 { + child.page = (*page)(unsafe.Pointer(&value[bucketHeaderSize])) + } + + return &child +} + +// CreateBucket creates a new bucket at the given key and returns the new bucket. +// Returns an error if the key already exists, if the bucket name is blank, or if the bucket name is too long. +// The bucket instance is only valid for the lifetime of the transaction. +func (b *Bucket) CreateBucket(key []byte) (*Bucket, error) { + if b.tx.db == nil { + return nil, ErrTxClosed + } else if !b.tx.writable { + return nil, ErrTxNotWritable + } else if len(key) == 0 { + return nil, ErrBucketNameRequired + } + + // Move cursor to correct position. + c := b.Cursor() + k, _, flags := c.seek(key) + + // Return an error if there is an existing key. + if bytes.Equal(key, k) { + if (flags & bucketLeafFlag) != 0 { + return nil, ErrBucketExists + } else { + return nil, ErrIncompatibleValue + } + } + + // Create empty, inline bucket. + var bucket = Bucket{ + bucket: &bucket{}, + rootNode: &node{isLeaf: true}, + FillPercent: DefaultFillPercent, + } + var value = bucket.write() + + // Insert into node. + key = cloneBytes(key) + c.node().put(key, key, value, 0, bucketLeafFlag) + + // Since subbuckets are not allowed on inline buckets, we need to + // dereference the inline page, if it exists. This will cause the bucket + // to be treated as a regular, non-inline bucket for the rest of the tx. + b.page = nil + + return b.Bucket(key), nil +} + +// CreateBucketIfNotExists creates a new bucket if it doesn't already exist and returns a reference to it. +// Returns an error if the bucket name is blank, or if the bucket name is too long. +// The bucket instance is only valid for the lifetime of the transaction. +func (b *Bucket) CreateBucketIfNotExists(key []byte) (*Bucket, error) { + child, err := b.CreateBucket(key) + if err == ErrBucketExists { + return b.Bucket(key), nil + } else if err != nil { + return nil, err + } + return child, nil +} + +// DeleteBucket deletes a bucket at the given key. +// Returns an error if the bucket does not exists, or if the key represents a non-bucket value. +func (b *Bucket) DeleteBucket(key []byte) error { + if b.tx.db == nil { + return ErrTxClosed + } else if !b.Writable() { + return ErrTxNotWritable + } + + // Move cursor to correct position. + c := b.Cursor() + k, _, flags := c.seek(key) + + // Return an error if bucket doesn't exist or is not a bucket. + if !bytes.Equal(key, k) { + return ErrBucketNotFound + } else if (flags & bucketLeafFlag) == 0 { + return ErrIncompatibleValue + } + + // Recursively delete all child buckets. + child := b.Bucket(key) + err := child.ForEach(func(k, v []byte) error { + if v == nil { + if err := child.DeleteBucket(k); err != nil { + return fmt.Errorf("delete bucket: %s", err) + } + } + return nil + }) + if err != nil { + return err + } + + // Remove cached copy. + delete(b.buckets, string(key)) + + // Release all bucket pages to freelist. + child.nodes = nil + child.rootNode = nil + child.free() + + // Delete the node if we have a matching key. + c.node().del(key) + + return nil +} + +// Get retrieves the value for a key in the bucket. +// Returns a nil value if the key does not exist or if the key is a nested bucket. +// The returned value is only valid for the life of the transaction. +func (b *Bucket) Get(key []byte) []byte { + k, v, flags := b.Cursor().seek(key) + + // Return nil if this is a bucket. + if (flags & bucketLeafFlag) != 0 { + return nil + } + + // If our target node isn't the same key as what's passed in then return nil. + if !bytes.Equal(key, k) { + return nil + } + return v +} + +// Put sets the value for a key in the bucket. +// If the key exist then its previous value will be overwritten. +// Supplied value must remain valid for the life of the transaction. +// Returns an error if the bucket was created from a read-only transaction, if the key is blank, if the key is too large, or if the value is too large. +func (b *Bucket) Put(key []byte, value []byte) error { + if b.tx.db == nil { + return ErrTxClosed + } else if !b.Writable() { + return ErrTxNotWritable + } else if len(key) == 0 { + return ErrKeyRequired + } else if len(key) > MaxKeySize { + return ErrKeyTooLarge + } else if int64(len(value)) > MaxValueSize { + return ErrValueTooLarge + } + + // Move cursor to correct position. + c := b.Cursor() + k, _, flags := c.seek(key) + + // Return an error if there is an existing key with a bucket value. + if bytes.Equal(key, k) && (flags&bucketLeafFlag) != 0 { + return ErrIncompatibleValue + } + + // Insert into node. + key = cloneBytes(key) + c.node().put(key, key, value, 0, 0) + + return nil +} + +// Delete removes a key from the bucket. +// If the key does not exist then nothing is done and a nil error is returned. +// Returns an error if the bucket was created from a read-only transaction. +func (b *Bucket) Delete(key []byte) error { + if b.tx.db == nil { + return ErrTxClosed + } else if !b.Writable() { + return ErrTxNotWritable + } + + // Move cursor to correct position. + c := b.Cursor() + _, _, flags := c.seek(key) + + // Return an error if there is already existing bucket value. + if (flags & bucketLeafFlag) != 0 { + return ErrIncompatibleValue + } + + // Delete the node if we have a matching key. + c.node().del(key) + + return nil +} + +// NextSequence returns an autoincrementing integer for the bucket. +func (b *Bucket) NextSequence() (uint64, error) { + if b.tx.db == nil { + return 0, ErrTxClosed + } else if !b.Writable() { + return 0, ErrTxNotWritable + } + + // Materialize the root node if it hasn't been already so that the + // bucket will be saved during commit. + if b.rootNode == nil { + _ = b.node(b.root, nil) + } + + // Increment and return the sequence. + b.bucket.sequence++ + return b.bucket.sequence, nil +} + +// ForEach executes a function for each key/value pair in a bucket. +// If the provided function returns an error then the iteration is stopped and +// the error is returned to the caller. The provided function must not modify +// the bucket; this will result in undefined behavior. +func (b *Bucket) ForEach(fn func(k, v []byte) error) error { + if b.tx.db == nil { + return ErrTxClosed + } + c := b.Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + if err := fn(k, v); err != nil { + return err + } + } + return nil +} + +// Stat returns stats on a bucket. +func (b *Bucket) Stats() BucketStats { + var s, subStats BucketStats + pageSize := b.tx.db.pageSize + s.BucketN += 1 + if b.root == 0 { + s.InlineBucketN += 1 + } + b.forEachPage(func(p *page, depth int) { + if (p.flags & leafPageFlag) != 0 { + s.KeyN += int(p.count) + + // used totals the used bytes for the page + used := pageHeaderSize + + if p.count != 0 { + // If page has any elements, add all element headers. + used += leafPageElementSize * int(p.count-1) + + // Add all element key, value sizes. + // The computation takes advantage of the fact that the position + // of the last element's key/value equals to the total of the sizes + // of all previous elements' keys and values. + // It also includes the last element's header. + lastElement := p.leafPageElement(p.count - 1) + used += int(lastElement.pos + lastElement.ksize + lastElement.vsize) + } + + if b.root == 0 { + // For inlined bucket just update the inline stats + s.InlineBucketInuse += used + } else { + // For non-inlined bucket update all the leaf stats + s.LeafPageN++ + s.LeafInuse += used + s.LeafOverflowN += int(p.overflow) + + // Collect stats from sub-buckets. + // Do that by iterating over all element headers + // looking for the ones with the bucketLeafFlag. + for i := uint16(0); i < p.count; i++ { + e := p.leafPageElement(i) + if (e.flags & bucketLeafFlag) != 0 { + // For any bucket element, open the element value + // and recursively call Stats on the contained bucket. + subStats.Add(b.openBucket(e.value()).Stats()) + } + } + } + } else if (p.flags & branchPageFlag) != 0 { + s.BranchPageN++ + lastElement := p.branchPageElement(p.count - 1) + + // used totals the used bytes for the page + // Add header and all element headers. + used := pageHeaderSize + (branchPageElementSize * int(p.count-1)) + + // Add size of all keys and values. + // Again, use the fact that last element's position equals to + // the total of key, value sizes of all previous elements. + used += int(lastElement.pos + lastElement.ksize) + s.BranchInuse += used + s.BranchOverflowN += int(p.overflow) + } + + // Keep track of maximum page depth. + if depth+1 > s.Depth { + s.Depth = (depth + 1) + } + }) + + // Alloc stats can be computed from page counts and pageSize. + s.BranchAlloc = (s.BranchPageN + s.BranchOverflowN) * pageSize + s.LeafAlloc = (s.LeafPageN + s.LeafOverflowN) * pageSize + + // Add the max depth of sub-buckets to get total nested depth. + s.Depth += subStats.Depth + // Add the stats for all sub-buckets + s.Add(subStats) + return s +} + +// forEachPage iterates over every page in a bucket, including inline pages. +func (b *Bucket) forEachPage(fn func(*page, int)) { + // If we have an inline page then just use that. + if b.page != nil { + fn(b.page, 0) + return + } + + // Otherwise traverse the page hierarchy. + b.tx.forEachPage(b.root, 0, fn) +} + +// forEachPageNode iterates over every page (or node) in a bucket. +// This also includes inline pages. +func (b *Bucket) forEachPageNode(fn func(*page, *node, int)) { + // If we have an inline page or root node then just use that. + if b.page != nil { + fn(b.page, nil, 0) + return + } + b._forEachPageNode(b.root, 0, fn) +} + +func (b *Bucket) _forEachPageNode(pgid pgid, depth int, fn func(*page, *node, int)) { + var p, n = b.pageNode(pgid) + + // Execute function. + fn(p, n, depth) + + // Recursively loop over children. + if p != nil { + if (p.flags & branchPageFlag) != 0 { + for i := 0; i < int(p.count); i++ { + elem := p.branchPageElement(uint16(i)) + b._forEachPageNode(elem.pgid, depth+1, fn) + } + } + } else { + if !n.isLeaf { + for _, inode := range n.inodes { + b._forEachPageNode(inode.pgid, depth+1, fn) + } + } + } +} + +// spill writes all the nodes for this bucket to dirty pages. +func (b *Bucket) spill() error { + // Spill all child buckets first. + for name, child := range b.buckets { + // If the child bucket is small enough and it has no child buckets then + // write it inline into the parent bucket's page. Otherwise spill it + // like a normal bucket and make the parent value a pointer to the page. + var value []byte + if child.inlineable() { + child.free() + value = child.write() + } else { + if err := child.spill(); err != nil { + return err + } + + // Update the child bucket header in this bucket. + value = make([]byte, unsafe.Sizeof(bucket{})) + var bucket = (*bucket)(unsafe.Pointer(&value[0])) + *bucket = *child.bucket + } + + // Skip writing the bucket if there are no materialized nodes. + if child.rootNode == nil { + continue + } + + // Update parent node. + var c = b.Cursor() + k, _, flags := c.seek([]byte(name)) + if !bytes.Equal([]byte(name), k) { + panic(fmt.Sprintf("misplaced bucket header: %x -> %x", []byte(name), k)) + } + if flags&bucketLeafFlag == 0 { + panic(fmt.Sprintf("unexpected bucket header flag: %x", flags)) + } + c.node().put([]byte(name), []byte(name), value, 0, bucketLeafFlag) + } + + // Ignore if there's not a materialized root node. + if b.rootNode == nil { + return nil + } + + // Spill nodes. + if err := b.rootNode.spill(); err != nil { + return err + } + b.rootNode = b.rootNode.root() + + // Update the root node for this bucket. + if b.rootNode.pgid >= b.tx.meta.pgid { + panic(fmt.Sprintf("pgid (%d) above high water mark (%d)", b.rootNode.pgid, b.tx.meta.pgid)) + } + b.root = b.rootNode.pgid + + return nil +} + +// inlineable returns true if a bucket is small enough to be written inline +// and if it contains no subbuckets. Otherwise returns false. +func (b *Bucket) inlineable() bool { + var n = b.rootNode + + // Bucket must only contain a single leaf node. + if n == nil || !n.isLeaf { + return false + } + + // Bucket is not inlineable if it contains subbuckets or if it goes beyond + // our threshold for inline bucket size. + var size = pageHeaderSize + for _, inode := range n.inodes { + size += leafPageElementSize + len(inode.key) + len(inode.value) + + if inode.flags&bucketLeafFlag != 0 { + return false + } else if size > b.maxInlineBucketSize() { + return false + } + } + + return true +} + +// Returns the maximum total size of a bucket to make it a candidate for inlining. +func (b *Bucket) maxInlineBucketSize() int { + return b.tx.db.pageSize / 4 +} + +// write allocates and writes a bucket to a byte slice. +func (b *Bucket) write() []byte { + // Allocate the appropriate size. + var n = b.rootNode + var value = make([]byte, bucketHeaderSize+n.size()) + + // Write a bucket header. + var bucket = (*bucket)(unsafe.Pointer(&value[0])) + *bucket = *b.bucket + + // Convert byte slice to a fake page and write the root node. + var p = (*page)(unsafe.Pointer(&value[bucketHeaderSize])) + n.write(p) + + return value +} + +// rebalance attempts to balance all nodes. +func (b *Bucket) rebalance() { + for _, n := range b.nodes { + n.rebalance() + } + for _, child := range b.buckets { + child.rebalance() + } +} + +// node creates a node from a page and associates it with a given parent. +func (b *Bucket) node(pgid pgid, parent *node) *node { + _assert(b.nodes != nil, "nodes map expected") + + // Retrieve node if it's already been created. + if n := b.nodes[pgid]; n != nil { + return n + } + + // Otherwise create a node and cache it. + n := &node{bucket: b, parent: parent} + if parent == nil { + b.rootNode = n + } else { + parent.children = append(parent.children, n) + } + + // Use the inline page if this is an inline bucket. + var p = b.page + if p == nil { + p = b.tx.page(pgid) + } + + // Read the page into the node and cache it. + n.read(p) + b.nodes[pgid] = n + + // Update statistics. + b.tx.stats.NodeCount++ + + return n +} + +// free recursively frees all pages in the bucket. +func (b *Bucket) free() { + if b.root == 0 { + return + } + + var tx = b.tx + b.forEachPageNode(func(p *page, n *node, _ int) { + if p != nil { + tx.db.freelist.free(tx.meta.txid, p) + } else { + n.free() + } + }) + b.root = 0 +} + +// dereference removes all references to the old mmap. +func (b *Bucket) dereference() { + if b.rootNode != nil { + b.rootNode.root().dereference() + } + + for _, child := range b.buckets { + child.dereference() + } +} + +// pageNode returns the in-memory node, if it exists. +// Otherwise returns the underlying page. +func (b *Bucket) pageNode(id pgid) (*page, *node) { + // Inline buckets have a fake page embedded in their value so treat them + // differently. We'll return the rootNode (if available) or the fake page. + if b.root == 0 { + if id != 0 { + panic(fmt.Sprintf("inline bucket non-zero page access(2): %d != 0", id)) + } + if b.rootNode != nil { + return nil, b.rootNode + } + return b.page, nil + } + + // Check the node cache for non-inline buckets. + if b.nodes != nil { + if n := b.nodes[id]; n != nil { + return nil, n + } + } + + // Finally lookup the page from the transaction if no node is materialized. + return b.tx.page(id), nil +} + +// BucketStats records statistics about resources used by a bucket. +type BucketStats struct { + // Page count statistics. + BranchPageN int // number of logical branch pages + BranchOverflowN int // number of physical branch overflow pages + LeafPageN int // number of logical leaf pages + LeafOverflowN int // number of physical leaf overflow pages + + // Tree statistics. + KeyN int // number of keys/value pairs + Depth int // number of levels in B+tree + + // Page size utilization. + BranchAlloc int // bytes allocated for physical branch pages + BranchInuse int // bytes actually used for branch data + LeafAlloc int // bytes allocated for physical leaf pages + LeafInuse int // bytes actually used for leaf data + + // Bucket statistics + BucketN int // total number of buckets including the top bucket + InlineBucketN int // total number on inlined buckets + InlineBucketInuse int // bytes used for inlined buckets (also accounted for in LeafInuse) +} + +func (s *BucketStats) Add(other BucketStats) { + s.BranchPageN += other.BranchPageN + s.BranchOverflowN += other.BranchOverflowN + s.LeafPageN += other.LeafPageN + s.LeafOverflowN += other.LeafOverflowN + s.KeyN += other.KeyN + if s.Depth < other.Depth { + s.Depth = other.Depth + } + s.BranchAlloc += other.BranchAlloc + s.BranchInuse += other.BranchInuse + s.LeafAlloc += other.LeafAlloc + s.LeafInuse += other.LeafInuse + + s.BucketN += other.BucketN + s.InlineBucketN += other.InlineBucketN + s.InlineBucketInuse += other.InlineBucketInuse +} + +// cloneBytes returns a copy of a given slice. +func cloneBytes(v []byte) []byte { + var clone = make([]byte, len(v)) + copy(clone, v) + return clone +} diff --git a/vendor/github.com/coreos/bbolt/bucket_test.go b/vendor/github.com/coreos/bbolt/bucket_test.go new file mode 100644 index 0000000..528fec2 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bucket_test.go @@ -0,0 +1,1867 @@ +package bolt_test + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "log" + "math/rand" + "os" + "strconv" + "strings" + "testing" + "testing/quick" + + "github.com/boltdb/bolt" +) + +// Ensure that a bucket that gets a non-existent key returns nil. +func TestBucket_Get_NonExistent(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if v := b.Get([]byte("foo")); v != nil { + t.Fatal("expected nil value") + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a bucket can read a value that is not flushed yet. +func TestBucket_Get_FromNode(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + if v := b.Get([]byte("foo")); !bytes.Equal(v, []byte("bar")) { + t.Fatalf("unexpected value: %v", v) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a bucket retrieved via Get() returns a nil. +func TestBucket_Get_IncompatibleValue(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + + if _, err := tx.Bucket([]byte("widgets")).CreateBucket([]byte("foo")); err != nil { + t.Fatal(err) + } + + if tx.Bucket([]byte("widgets")).Get([]byte("foo")) != nil { + t.Fatal("expected nil value") + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a slice returned from a bucket has a capacity equal to its length. +// This also allows slices to be appended to since it will require a realloc by Go. +// +// https://github.com/boltdb/bolt/issues/544 +func TestBucket_Get_Capacity(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + // Write key to a bucket. + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("bucket")) + if err != nil { + return err + } + return b.Put([]byte("key"), []byte("val")) + }); err != nil { + t.Fatal(err) + } + + // Retrieve value and attempt to append to it. + if err := db.Update(func(tx *bolt.Tx) error { + k, v := tx.Bucket([]byte("bucket")).Cursor().First() + + // Verify capacity. + if len(k) != cap(k) { + t.Fatalf("unexpected key slice capacity: %d", cap(k)) + } else if len(v) != cap(v) { + t.Fatalf("unexpected value slice capacity: %d", cap(v)) + } + + // Ensure slice can be appended to without a segfault. + k = append(k, []byte("123")...) + v = append(v, []byte("123")...) + + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a bucket can write a key/value. +func TestBucket_Put(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + + v := tx.Bucket([]byte("widgets")).Get([]byte("foo")) + if !bytes.Equal([]byte("bar"), v) { + t.Fatalf("unexpected value: %v", v) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a bucket can rewrite a key in the same transaction. +func TestBucket_Put_Repeat(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte("baz")); err != nil { + t.Fatal(err) + } + + value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) + if !bytes.Equal([]byte("baz"), value) { + t.Fatalf("unexpected value: %v", value) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a bucket can write a bunch of large values. +func TestBucket_Put_Large(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + count, factor := 100, 200 + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + for i := 1; i < count; i++ { + if err := b.Put([]byte(strings.Repeat("0", i*factor)), []byte(strings.Repeat("X", (count-i)*factor))); err != nil { + t.Fatal(err) + } + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("widgets")) + for i := 1; i < count; i++ { + value := b.Get([]byte(strings.Repeat("0", i*factor))) + if !bytes.Equal(value, []byte(strings.Repeat("X", (count-i)*factor))) { + t.Fatalf("unexpected value: %v", value) + } + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a database can perform multiple large appends safely. +func TestDB_Put_VeryLarge(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + n, batchN := 400000, 200000 + ksize, vsize := 8, 500 + + db := MustOpenDB() + defer db.MustClose() + + for i := 0; i < n; i += batchN { + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + for j := 0; j < batchN; j++ { + k, v := make([]byte, ksize), make([]byte, vsize) + binary.BigEndian.PutUint32(k, uint32(i+j)) + if err := b.Put(k, v); err != nil { + t.Fatal(err) + } + } + return nil + }); err != nil { + t.Fatal(err) + } + } +} + +// Ensure that a setting a value on a key with a bucket value returns an error. +func TestBucket_Put_IncompatibleValue(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + b0, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + + if _, err := tx.Bucket([]byte("widgets")).CreateBucket([]byte("foo")); err != nil { + t.Fatal(err) + } + if err := b0.Put([]byte("foo"), []byte("bar")); err != bolt.ErrIncompatibleValue { + t.Fatalf("unexpected error: %s", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a setting a value while the transaction is closed returns an error. +func TestBucket_Put_Closed(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + tx, err := db.Begin(true) + if err != nil { + t.Fatal(err) + } + + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + + if err := b.Put([]byte("foo"), []byte("bar")); err != bolt.ErrTxClosed { + t.Fatalf("unexpected error: %s", err) + } +} + +// Ensure that setting a value on a read-only bucket returns an error. +func TestBucket_Put_ReadOnly(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("widgets")) + if err := b.Put([]byte("foo"), []byte("bar")); err != bolt.ErrTxNotWritable { + t.Fatalf("unexpected error: %s", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a bucket can delete an existing key. +func TestBucket_Delete(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + if err := b.Delete([]byte("foo")); err != nil { + t.Fatal(err) + } + if v := b.Get([]byte("foo")); v != nil { + t.Fatalf("unexpected value: %v", v) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that deleting a large set of keys will work correctly. +func TestBucket_Delete_Large(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < 100; i++ { + if err := b.Put([]byte(strconv.Itoa(i)), []byte(strings.Repeat("*", 1024))); err != nil { + t.Fatal(err) + } + } + + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("widgets")) + for i := 0; i < 100; i++ { + if err := b.Delete([]byte(strconv.Itoa(i))); err != nil { + t.Fatal(err) + } + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("widgets")) + for i := 0; i < 100; i++ { + if v := b.Get([]byte(strconv.Itoa(i))); v != nil { + t.Fatalf("unexpected value: %v, i=%d", v, i) + } + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Deleting a very large list of keys will cause the freelist to use overflow. +func TestBucket_Delete_FreelistOverflow(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + db := MustOpenDB() + defer db.MustClose() + + k := make([]byte, 16) + for i := uint64(0); i < 10000; i++ { + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte("0")) + if err != nil { + t.Fatalf("bucket error: %s", err) + } + + for j := uint64(0); j < 1000; j++ { + binary.BigEndian.PutUint64(k[:8], i) + binary.BigEndian.PutUint64(k[8:], j) + if err := b.Put(k, nil); err != nil { + t.Fatalf("put error: %s", err) + } + } + + return nil + }); err != nil { + t.Fatal(err) + } + } + + // Delete all of them in one large transaction + if err := db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("0")) + c := b.Cursor() + for k, _ := c.First(); k != nil; k, _ = c.Next() { + if err := c.Delete(); err != nil { + t.Fatal(err) + } + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that accessing and updating nested buckets is ok across transactions. +func TestBucket_Nested(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + // Create a widgets bucket. + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + + // Create a widgets/foo bucket. + _, err = b.CreateBucket([]byte("foo")) + if err != nil { + t.Fatal(err) + } + + // Create a widgets/bar key. + if err := b.Put([]byte("bar"), []byte("0000")); err != nil { + t.Fatal(err) + } + + return nil + }); err != nil { + t.Fatal(err) + } + db.MustCheck() + + // Update widgets/bar. + if err := db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("widgets")) + if err := b.Put([]byte("bar"), []byte("xxxx")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + db.MustCheck() + + // Cause a split. + if err := db.Update(func(tx *bolt.Tx) error { + var b = tx.Bucket([]byte("widgets")) + for i := 0; i < 10000; i++ { + if err := b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil { + t.Fatal(err) + } + } + return nil + }); err != nil { + t.Fatal(err) + } + db.MustCheck() + + // Insert into widgets/foo/baz. + if err := db.Update(func(tx *bolt.Tx) error { + var b = tx.Bucket([]byte("widgets")) + if err := b.Bucket([]byte("foo")).Put([]byte("baz"), []byte("yyyy")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + db.MustCheck() + + // Verify. + if err := db.View(func(tx *bolt.Tx) error { + var b = tx.Bucket([]byte("widgets")) + if v := b.Bucket([]byte("foo")).Get([]byte("baz")); !bytes.Equal(v, []byte("yyyy")) { + t.Fatalf("unexpected value: %v", v) + } + if v := b.Get([]byte("bar")); !bytes.Equal(v, []byte("xxxx")) { + t.Fatalf("unexpected value: %v", v) + } + for i := 0; i < 10000; i++ { + if v := b.Get([]byte(strconv.Itoa(i))); !bytes.Equal(v, []byte(strconv.Itoa(i))) { + t.Fatalf("unexpected value: %v", v) + } + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that deleting a bucket using Delete() returns an error. +func TestBucket_Delete_Bucket(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if _, err := b.CreateBucket([]byte("foo")); err != nil { + t.Fatal(err) + } + if err := b.Delete([]byte("foo")); err != bolt.ErrIncompatibleValue { + t.Fatalf("unexpected error: %s", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that deleting a key on a read-only bucket returns an error. +func TestBucket_Delete_ReadOnly(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.View(func(tx *bolt.Tx) error { + if err := tx.Bucket([]byte("widgets")).Delete([]byte("foo")); err != bolt.ErrTxNotWritable { + t.Fatalf("unexpected error: %s", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a deleting value while the transaction is closed returns an error. +func TestBucket_Delete_Closed(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + tx, err := db.Begin(true) + if err != nil { + t.Fatal(err) + } + + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + if err := b.Delete([]byte("foo")); err != bolt.ErrTxClosed { + t.Fatalf("unexpected error: %s", err) + } +} + +// Ensure that deleting a bucket causes nested buckets to be deleted. +func TestBucket_DeleteBucket_Nested(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + widgets, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + + foo, err := widgets.CreateBucket([]byte("foo")) + if err != nil { + t.Fatal(err) + } + + bar, err := foo.CreateBucket([]byte("bar")) + if err != nil { + t.Fatal(err) + } + if err := bar.Put([]byte("baz"), []byte("bat")); err != nil { + t.Fatal(err) + } + if err := tx.Bucket([]byte("widgets")).DeleteBucket([]byte("foo")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that deleting a bucket causes nested buckets to be deleted after they have been committed. +func TestBucket_DeleteBucket_Nested2(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + widgets, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + + foo, err := widgets.CreateBucket([]byte("foo")) + if err != nil { + t.Fatal(err) + } + + bar, err := foo.CreateBucket([]byte("bar")) + if err != nil { + t.Fatal(err) + } + + if err := bar.Put([]byte("baz"), []byte("bat")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.Update(func(tx *bolt.Tx) error { + widgets := tx.Bucket([]byte("widgets")) + if widgets == nil { + t.Fatal("expected widgets bucket") + } + + foo := widgets.Bucket([]byte("foo")) + if foo == nil { + t.Fatal("expected foo bucket") + } + + bar := foo.Bucket([]byte("bar")) + if bar == nil { + t.Fatal("expected bar bucket") + } + + if v := bar.Get([]byte("baz")); !bytes.Equal(v, []byte("bat")) { + t.Fatalf("unexpected value: %v", v) + } + if err := tx.DeleteBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.View(func(tx *bolt.Tx) error { + if tx.Bucket([]byte("widgets")) != nil { + t.Fatal("expected bucket to be deleted") + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that deleting a child bucket with multiple pages causes all pages to get collected. +// NOTE: Consistency check in bolt_test.DB.Close() will panic if pages not freed properly. +func TestBucket_DeleteBucket_Large(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + widgets, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + + foo, err := widgets.CreateBucket([]byte("foo")) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < 1000; i++ { + if err := foo.Put([]byte(fmt.Sprintf("%d", i)), []byte(fmt.Sprintf("%0100d", i))); err != nil { + t.Fatal(err) + } + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.Update(func(tx *bolt.Tx) error { + if err := tx.DeleteBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a simple value retrieved via Bucket() returns a nil. +func TestBucket_Bucket_IncompatibleValue(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + widgets, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + + if err := widgets.Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + if b := tx.Bucket([]byte("widgets")).Bucket([]byte("foo")); b != nil { + t.Fatal("expected nil bucket") + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that creating a bucket on an existing non-bucket key returns an error. +func TestBucket_CreateBucket_IncompatibleValue(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + widgets, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + + if err := widgets.Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + if _, err := widgets.CreateBucket([]byte("foo")); err != bolt.ErrIncompatibleValue { + t.Fatalf("unexpected error: %s", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that deleting a bucket on an existing non-bucket key returns an error. +func TestBucket_DeleteBucket_IncompatibleValue(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + widgets, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := widgets.Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + if err := tx.Bucket([]byte("widgets")).DeleteBucket([]byte("foo")); err != bolt.ErrIncompatibleValue { + t.Fatalf("unexpected error: %s", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a bucket can return an autoincrementing sequence. +func TestBucket_NextSequence(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + widgets, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + woojits, err := tx.CreateBucket([]byte("woojits")) + if err != nil { + t.Fatal(err) + } + + // Make sure sequence increments. + if seq, err := widgets.NextSequence(); err != nil { + t.Fatal(err) + } else if seq != 1 { + t.Fatalf("unexpecte sequence: %d", seq) + } + + if seq, err := widgets.NextSequence(); err != nil { + t.Fatal(err) + } else if seq != 2 { + t.Fatalf("unexpected sequence: %d", seq) + } + + // Buckets should be separate. + if seq, err := woojits.NextSequence(); err != nil { + t.Fatal(err) + } else if seq != 1 { + t.Fatalf("unexpected sequence: %d", 1) + } + + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a bucket will persist an autoincrementing sequence even if its +// the only thing updated on the bucket. +// https://github.com/boltdb/bolt/issues/296 +func TestBucket_NextSequence_Persist(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.Bucket([]byte("widgets")).NextSequence(); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.Update(func(tx *bolt.Tx) error { + seq, err := tx.Bucket([]byte("widgets")).NextSequence() + if err != nil { + t.Fatalf("unexpected error: %s", err) + } else if seq != 2 { + t.Fatalf("unexpected sequence: %d", seq) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that retrieving the next sequence on a read-only bucket returns an error. +func TestBucket_NextSequence_ReadOnly(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.View(func(tx *bolt.Tx) error { + _, err := tx.Bucket([]byte("widgets")).NextSequence() + if err != bolt.ErrTxNotWritable { + t.Fatalf("unexpected error: %s", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that retrieving the next sequence for a bucket on a closed database return an error. +func TestBucket_NextSequence_Closed(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + tx, err := db.Begin(true) + if err != nil { + t.Fatal(err) + } + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + if _, err := b.NextSequence(); err != bolt.ErrTxClosed { + t.Fatal(err) + } +} + +// Ensure a user can loop over all key/value pairs in a bucket. +func TestBucket_ForEach(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte("0000")); err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("baz"), []byte("0001")); err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("bar"), []byte("0002")); err != nil { + t.Fatal(err) + } + + var index int + if err := b.ForEach(func(k, v []byte) error { + switch index { + case 0: + if !bytes.Equal(k, []byte("bar")) { + t.Fatalf("unexpected key: %v", k) + } else if !bytes.Equal(v, []byte("0002")) { + t.Fatalf("unexpected value: %v", v) + } + case 1: + if !bytes.Equal(k, []byte("baz")) { + t.Fatalf("unexpected key: %v", k) + } else if !bytes.Equal(v, []byte("0001")) { + t.Fatalf("unexpected value: %v", v) + } + case 2: + if !bytes.Equal(k, []byte("foo")) { + t.Fatalf("unexpected key: %v", k) + } else if !bytes.Equal(v, []byte("0000")) { + t.Fatalf("unexpected value: %v", v) + } + } + index++ + return nil + }); err != nil { + t.Fatal(err) + } + + if index != 3 { + t.Fatalf("unexpected index: %d", index) + } + + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure a database can stop iteration early. +func TestBucket_ForEach_ShortCircuit(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("bar"), []byte("0000")); err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("baz"), []byte("0000")); err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte("0000")); err != nil { + t.Fatal(err) + } + + var index int + if err := tx.Bucket([]byte("widgets")).ForEach(func(k, v []byte) error { + index++ + if bytes.Equal(k, []byte("baz")) { + return errors.New("marker") + } + return nil + }); err == nil || err.Error() != "marker" { + t.Fatalf("unexpected error: %s", err) + } + if index != 2 { + t.Fatalf("unexpected index: %d", index) + } + + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that looping over a bucket on a closed database returns an error. +func TestBucket_ForEach_Closed(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + tx, err := db.Begin(true) + if err != nil { + t.Fatal(err) + } + + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + + if err := b.ForEach(func(k, v []byte) error { return nil }); err != bolt.ErrTxClosed { + t.Fatalf("unexpected error: %s", err) + } +} + +// Ensure that an error is returned when inserting with an empty key. +func TestBucket_Put_EmptyKey(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte(""), []byte("bar")); err != bolt.ErrKeyRequired { + t.Fatalf("unexpected error: %s", err) + } + if err := b.Put(nil, []byte("bar")); err != bolt.ErrKeyRequired { + t.Fatalf("unexpected error: %s", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that an error is returned when inserting with a key that's too large. +func TestBucket_Put_KeyTooLarge(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put(make([]byte, 32769), []byte("bar")); err != bolt.ErrKeyTooLarge { + t.Fatalf("unexpected error: %s", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that an error is returned when inserting a value that's too large. +func TestBucket_Put_ValueTooLarge(t *testing.T) { + // Skip this test on DroneCI because the machine is resource constrained. + if os.Getenv("DRONE") == "true" { + t.Skip("not enough RAM for test") + } + + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), make([]byte, bolt.MaxValueSize+1)); err != bolt.ErrValueTooLarge { + t.Fatalf("unexpected error: %s", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure a bucket can calculate stats. +func TestBucket_Stats(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + // Add bucket with fewer keys but one big value. + bigKey := []byte("really-big-value") + for i := 0; i < 500; i++ { + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte("woojits")) + if err != nil { + t.Fatal(err) + } + + if err := b.Put([]byte(fmt.Sprintf("%03d", i)), []byte(strconv.Itoa(i))); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + } + if err := db.Update(func(tx *bolt.Tx) error { + if err := tx.Bucket([]byte("woojits")).Put(bigKey, []byte(strings.Repeat("*", 10000))); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + db.MustCheck() + + if err := db.View(func(tx *bolt.Tx) error { + stats := tx.Bucket([]byte("woojits")).Stats() + if stats.BranchPageN != 1 { + t.Fatalf("unexpected BranchPageN: %d", stats.BranchPageN) + } else if stats.BranchOverflowN != 0 { + t.Fatalf("unexpected BranchOverflowN: %d", stats.BranchOverflowN) + } else if stats.LeafPageN != 7 { + t.Fatalf("unexpected LeafPageN: %d", stats.LeafPageN) + } else if stats.LeafOverflowN != 2 { + t.Fatalf("unexpected LeafOverflowN: %d", stats.LeafOverflowN) + } else if stats.KeyN != 501 { + t.Fatalf("unexpected KeyN: %d", stats.KeyN) + } else if stats.Depth != 2 { + t.Fatalf("unexpected Depth: %d", stats.Depth) + } + + branchInuse := 16 // branch page header + branchInuse += 7 * 16 // branch elements + branchInuse += 7 * 3 // branch keys (6 3-byte keys) + if stats.BranchInuse != branchInuse { + t.Fatalf("unexpected BranchInuse: %d", stats.BranchInuse) + } + + leafInuse := 7 * 16 // leaf page header + leafInuse += 501 * 16 // leaf elements + leafInuse += 500*3 + len(bigKey) // leaf keys + leafInuse += 1*10 + 2*90 + 3*400 + 10000 // leaf values + if stats.LeafInuse != leafInuse { + t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse) + } + + // Only check allocations for 4KB pages. + if os.Getpagesize() == 4096 { + if stats.BranchAlloc != 4096 { + t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) + } else if stats.LeafAlloc != 36864 { + t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) + } + } + + if stats.BucketN != 1 { + t.Fatalf("unexpected BucketN: %d", stats.BucketN) + } else if stats.InlineBucketN != 0 { + t.Fatalf("unexpected InlineBucketN: %d", stats.InlineBucketN) + } else if stats.InlineBucketInuse != 0 { + t.Fatalf("unexpected InlineBucketInuse: %d", stats.InlineBucketInuse) + } + + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure a bucket with random insertion utilizes fill percentage correctly. +func TestBucket_Stats_RandomFill(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } else if os.Getpagesize() != 4096 { + t.Skip("invalid page size for test") + } + + db := MustOpenDB() + defer db.MustClose() + + // Add a set of values in random order. It will be the same random + // order so we can maintain consistency between test runs. + var count int + rand := rand.New(rand.NewSource(42)) + for _, i := range rand.Perm(1000) { + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte("woojits")) + if err != nil { + t.Fatal(err) + } + b.FillPercent = 0.9 + for _, j := range rand.Perm(100) { + index := (j * 10000) + i + if err := b.Put([]byte(fmt.Sprintf("%d000000000000000", index)), []byte("0000000000")); err != nil { + t.Fatal(err) + } + count++ + } + return nil + }); err != nil { + t.Fatal(err) + } + } + + db.MustCheck() + + if err := db.View(func(tx *bolt.Tx) error { + stats := tx.Bucket([]byte("woojits")).Stats() + if stats.KeyN != 100000 { + t.Fatalf("unexpected KeyN: %d", stats.KeyN) + } + + if stats.BranchPageN != 98 { + t.Fatalf("unexpected BranchPageN: %d", stats.BranchPageN) + } else if stats.BranchOverflowN != 0 { + t.Fatalf("unexpected BranchOverflowN: %d", stats.BranchOverflowN) + } else if stats.BranchInuse != 130984 { + t.Fatalf("unexpected BranchInuse: %d", stats.BranchInuse) + } else if stats.BranchAlloc != 401408 { + t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) + } + + if stats.LeafPageN != 3412 { + t.Fatalf("unexpected LeafPageN: %d", stats.LeafPageN) + } else if stats.LeafOverflowN != 0 { + t.Fatalf("unexpected LeafOverflowN: %d", stats.LeafOverflowN) + } else if stats.LeafInuse != 4742482 { + t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse) + } else if stats.LeafAlloc != 13975552 { + t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure a bucket can calculate stats. +func TestBucket_Stats_Small(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + // Add a bucket that fits on a single root leaf. + b, err := tx.CreateBucket([]byte("whozawhats")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + + return nil + }); err != nil { + t.Fatal(err) + } + + db.MustCheck() + + if err := db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("whozawhats")) + stats := b.Stats() + if stats.BranchPageN != 0 { + t.Fatalf("unexpected BranchPageN: %d", stats.BranchPageN) + } else if stats.BranchOverflowN != 0 { + t.Fatalf("unexpected BranchOverflowN: %d", stats.BranchOverflowN) + } else if stats.LeafPageN != 0 { + t.Fatalf("unexpected LeafPageN: %d", stats.LeafPageN) + } else if stats.LeafOverflowN != 0 { + t.Fatalf("unexpected LeafOverflowN: %d", stats.LeafOverflowN) + } else if stats.KeyN != 1 { + t.Fatalf("unexpected KeyN: %d", stats.KeyN) + } else if stats.Depth != 1 { + t.Fatalf("unexpected Depth: %d", stats.Depth) + } else if stats.BranchInuse != 0 { + t.Fatalf("unexpected BranchInuse: %d", stats.BranchInuse) + } else if stats.LeafInuse != 0 { + t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse) + } + + if os.Getpagesize() == 4096 { + if stats.BranchAlloc != 0 { + t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) + } else if stats.LeafAlloc != 0 { + t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) + } + } + + if stats.BucketN != 1 { + t.Fatalf("unexpected BucketN: %d", stats.BucketN) + } else if stats.InlineBucketN != 1 { + t.Fatalf("unexpected InlineBucketN: %d", stats.InlineBucketN) + } else if stats.InlineBucketInuse != 16+16+6 { + t.Fatalf("unexpected InlineBucketInuse: %d", stats.InlineBucketInuse) + } + + return nil + }); err != nil { + t.Fatal(err) + } +} + +func TestBucket_Stats_EmptyBucket(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + // Add a bucket that fits on a single root leaf. + if _, err := tx.CreateBucket([]byte("whozawhats")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + db.MustCheck() + + if err := db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("whozawhats")) + stats := b.Stats() + if stats.BranchPageN != 0 { + t.Fatalf("unexpected BranchPageN: %d", stats.BranchPageN) + } else if stats.BranchOverflowN != 0 { + t.Fatalf("unexpected BranchOverflowN: %d", stats.BranchOverflowN) + } else if stats.LeafPageN != 0 { + t.Fatalf("unexpected LeafPageN: %d", stats.LeafPageN) + } else if stats.LeafOverflowN != 0 { + t.Fatalf("unexpected LeafOverflowN: %d", stats.LeafOverflowN) + } else if stats.KeyN != 0 { + t.Fatalf("unexpected KeyN: %d", stats.KeyN) + } else if stats.Depth != 1 { + t.Fatalf("unexpected Depth: %d", stats.Depth) + } else if stats.BranchInuse != 0 { + t.Fatalf("unexpected BranchInuse: %d", stats.BranchInuse) + } else if stats.LeafInuse != 0 { + t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse) + } + + if os.Getpagesize() == 4096 { + if stats.BranchAlloc != 0 { + t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) + } else if stats.LeafAlloc != 0 { + t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) + } + } + + if stats.BucketN != 1 { + t.Fatalf("unexpected BucketN: %d", stats.BucketN) + } else if stats.InlineBucketN != 1 { + t.Fatalf("unexpected InlineBucketN: %d", stats.InlineBucketN) + } else if stats.InlineBucketInuse != 16 { + t.Fatalf("unexpected InlineBucketInuse: %d", stats.InlineBucketInuse) + } + + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure a bucket can calculate stats. +func TestBucket_Stats_Nested(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("foo")) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 100; i++ { + if err := b.Put([]byte(fmt.Sprintf("%02d", i)), []byte(fmt.Sprintf("%02d", i))); err != nil { + t.Fatal(err) + } + } + + bar, err := b.CreateBucket([]byte("bar")) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 10; i++ { + if err := bar.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil { + t.Fatal(err) + } + } + + baz, err := bar.CreateBucket([]byte("baz")) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 10; i++ { + if err := baz.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil { + t.Fatal(err) + } + } + + return nil + }); err != nil { + t.Fatal(err) + } + + db.MustCheck() + + if err := db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("foo")) + stats := b.Stats() + if stats.BranchPageN != 0 { + t.Fatalf("unexpected BranchPageN: %d", stats.BranchPageN) + } else if stats.BranchOverflowN != 0 { + t.Fatalf("unexpected BranchOverflowN: %d", stats.BranchOverflowN) + } else if stats.LeafPageN != 2 { + t.Fatalf("unexpected LeafPageN: %d", stats.LeafPageN) + } else if stats.LeafOverflowN != 0 { + t.Fatalf("unexpected LeafOverflowN: %d", stats.LeafOverflowN) + } else if stats.KeyN != 122 { + t.Fatalf("unexpected KeyN: %d", stats.KeyN) + } else if stats.Depth != 3 { + t.Fatalf("unexpected Depth: %d", stats.Depth) + } else if stats.BranchInuse != 0 { + t.Fatalf("unexpected BranchInuse: %d", stats.BranchInuse) + } + + foo := 16 // foo (pghdr) + foo += 101 * 16 // foo leaf elements + foo += 100*2 + 100*2 // foo leaf key/values + foo += 3 + 16 // foo -> bar key/value + + bar := 16 // bar (pghdr) + bar += 11 * 16 // bar leaf elements + bar += 10 + 10 // bar leaf key/values + bar += 3 + 16 // bar -> baz key/value + + baz := 16 // baz (inline) (pghdr) + baz += 10 * 16 // baz leaf elements + baz += 10 + 10 // baz leaf key/values + + if stats.LeafInuse != foo+bar+baz { + t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse) + } + + if os.Getpagesize() == 4096 { + if stats.BranchAlloc != 0 { + t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) + } else if stats.LeafAlloc != 8192 { + t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) + } + } + + if stats.BucketN != 3 { + t.Fatalf("unexpected BucketN: %d", stats.BucketN) + } else if stats.InlineBucketN != 1 { + t.Fatalf("unexpected InlineBucketN: %d", stats.InlineBucketN) + } else if stats.InlineBucketInuse != baz { + t.Fatalf("unexpected InlineBucketInuse: %d", stats.InlineBucketInuse) + } + + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure a large bucket can calculate stats. +func TestBucket_Stats_Large(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + db := MustOpenDB() + defer db.MustClose() + + var index int + for i := 0; i < 100; i++ { + // Add bucket with lots of keys. + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 1000; i++ { + if err := b.Put([]byte(strconv.Itoa(index)), []byte(strconv.Itoa(index))); err != nil { + t.Fatal(err) + } + index++ + } + return nil + }); err != nil { + t.Fatal(err) + } + } + + db.MustCheck() + + if err := db.View(func(tx *bolt.Tx) error { + stats := tx.Bucket([]byte("widgets")).Stats() + if stats.BranchPageN != 13 { + t.Fatalf("unexpected BranchPageN: %d", stats.BranchPageN) + } else if stats.BranchOverflowN != 0 { + t.Fatalf("unexpected BranchOverflowN: %d", stats.BranchOverflowN) + } else if stats.LeafPageN != 1196 { + t.Fatalf("unexpected LeafPageN: %d", stats.LeafPageN) + } else if stats.LeafOverflowN != 0 { + t.Fatalf("unexpected LeafOverflowN: %d", stats.LeafOverflowN) + } else if stats.KeyN != 100000 { + t.Fatalf("unexpected KeyN: %d", stats.KeyN) + } else if stats.Depth != 3 { + t.Fatalf("unexpected Depth: %d", stats.Depth) + } else if stats.BranchInuse != 25257 { + t.Fatalf("unexpected BranchInuse: %d", stats.BranchInuse) + } else if stats.LeafInuse != 2596916 { + t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse) + } + + if os.Getpagesize() == 4096 { + if stats.BranchAlloc != 53248 { + t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) + } else if stats.LeafAlloc != 4898816 { + t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) + } + } + + if stats.BucketN != 1 { + t.Fatalf("unexpected BucketN: %d", stats.BucketN) + } else if stats.InlineBucketN != 0 { + t.Fatalf("unexpected InlineBucketN: %d", stats.InlineBucketN) + } else if stats.InlineBucketInuse != 0 { + t.Fatalf("unexpected InlineBucketInuse: %d", stats.InlineBucketInuse) + } + + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a bucket can write random keys and values across multiple transactions. +func TestBucket_Put_Single(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + index := 0 + if err := quick.Check(func(items testdata) bool { + db := MustOpenDB() + defer db.MustClose() + + m := make(map[string][]byte) + + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + for _, item := range items { + if err := db.Update(func(tx *bolt.Tx) error { + if err := tx.Bucket([]byte("widgets")).Put(item.Key, item.Value); err != nil { + panic("put error: " + err.Error()) + } + m[string(item.Key)] = item.Value + return nil + }); err != nil { + t.Fatal(err) + } + + // Verify all key/values so far. + if err := db.View(func(tx *bolt.Tx) error { + i := 0 + for k, v := range m { + value := tx.Bucket([]byte("widgets")).Get([]byte(k)) + if !bytes.Equal(value, v) { + t.Logf("value mismatch [run %d] (%d of %d):\nkey: %x\ngot: %x\nexp: %x", index, i, len(m), []byte(k), value, v) + db.CopyTempFile() + t.FailNow() + } + i++ + } + return nil + }); err != nil { + t.Fatal(err) + } + } + + index++ + return true + }, nil); err != nil { + t.Error(err) + } +} + +// Ensure that a transaction can insert multiple key/value pairs at once. +func TestBucket_Put_Multiple(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + if err := quick.Check(func(items testdata) bool { + db := MustOpenDB() + defer db.MustClose() + + // Bulk insert all values. + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("widgets")) + for _, item := range items { + if err := b.Put(item.Key, item.Value); err != nil { + t.Fatal(err) + } + } + return nil + }); err != nil { + t.Fatal(err) + } + + // Verify all items exist. + if err := db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("widgets")) + for _, item := range items { + value := b.Get(item.Key) + if !bytes.Equal(item.Value, value) { + db.CopyTempFile() + t.Fatalf("exp=%x; got=%x", item.Value, value) + } + } + return nil + }); err != nil { + t.Fatal(err) + } + + return true + }, qconfig()); err != nil { + t.Error(err) + } +} + +// Ensure that a transaction can delete all key/value pairs and return to a single leaf page. +func TestBucket_Delete_Quick(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + if err := quick.Check(func(items testdata) bool { + db := MustOpenDB() + defer db.MustClose() + + // Bulk insert all values. + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("widgets")) + for _, item := range items { + if err := b.Put(item.Key, item.Value); err != nil { + t.Fatal(err) + } + } + return nil + }); err != nil { + t.Fatal(err) + } + + // Remove items one at a time and check consistency. + for _, item := range items { + if err := db.Update(func(tx *bolt.Tx) error { + return tx.Bucket([]byte("widgets")).Delete(item.Key) + }); err != nil { + t.Fatal(err) + } + } + + // Anything before our deletion index should be nil. + if err := db.View(func(tx *bolt.Tx) error { + if err := tx.Bucket([]byte("widgets")).ForEach(func(k, v []byte) error { + t.Fatalf("bucket should be empty; found: %06x", trunc(k, 3)) + return nil + }); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + return true + }, qconfig()); err != nil { + t.Error(err) + } +} + +func ExampleBucket_Put() { + // Open the database. + db, err := bolt.Open(tempfile(), 0666, nil) + if err != nil { + log.Fatal(err) + } + defer os.Remove(db.Path()) + + // Start a write transaction. + if err := db.Update(func(tx *bolt.Tx) error { + // Create a bucket. + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + return err + } + + // Set the value "bar" for the key "foo". + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + return err + } + return nil + }); err != nil { + log.Fatal(err) + } + + // Read value back in a different read-only transaction. + if err := db.View(func(tx *bolt.Tx) error { + value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) + fmt.Printf("The value of 'foo' is: %s\n", value) + return nil + }); err != nil { + log.Fatal(err) + } + + // Close database to release file lock. + if err := db.Close(); err != nil { + log.Fatal(err) + } + + // Output: + // The value of 'foo' is: bar +} + +func ExampleBucket_Delete() { + // Open the database. + db, err := bolt.Open(tempfile(), 0666, nil) + if err != nil { + log.Fatal(err) + } + defer os.Remove(db.Path()) + + // Start a write transaction. + if err := db.Update(func(tx *bolt.Tx) error { + // Create a bucket. + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + return err + } + + // Set the value "bar" for the key "foo". + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + return err + } + + // Retrieve the key back from the database and verify it. + value := b.Get([]byte("foo")) + fmt.Printf("The value of 'foo' was: %s\n", value) + + return nil + }); err != nil { + log.Fatal(err) + } + + // Delete the key in a different write transaction. + if err := db.Update(func(tx *bolt.Tx) error { + return tx.Bucket([]byte("widgets")).Delete([]byte("foo")) + }); err != nil { + log.Fatal(err) + } + + // Retrieve the key again. + if err := db.View(func(tx *bolt.Tx) error { + value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) + if value == nil { + fmt.Printf("The value of 'foo' is now: nil\n") + } + return nil + }); err != nil { + log.Fatal(err) + } + + // Close database to release file lock. + if err := db.Close(); err != nil { + log.Fatal(err) + } + + // Output: + // The value of 'foo' was: bar + // The value of 'foo' is now: nil +} + +func ExampleBucket_ForEach() { + // Open the database. + db, err := bolt.Open(tempfile(), 0666, nil) + if err != nil { + log.Fatal(err) + } + defer os.Remove(db.Path()) + + // Insert data into a bucket. + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("animals")) + if err != nil { + return err + } + + if err := b.Put([]byte("dog"), []byte("fun")); err != nil { + return err + } + if err := b.Put([]byte("cat"), []byte("lame")); err != nil { + return err + } + if err := b.Put([]byte("liger"), []byte("awesome")); err != nil { + return err + } + + // Iterate over items in sorted key order. + if err := b.ForEach(func(k, v []byte) error { + fmt.Printf("A %s is %s.\n", k, v) + return nil + }); err != nil { + return err + } + + return nil + }); err != nil { + log.Fatal(err) + } + + // Close database to release file lock. + if err := db.Close(); err != nil { + log.Fatal(err) + } + + // Output: + // A cat is lame. + // A dog is fun. + // A liger is awesome. +} diff --git a/vendor/github.com/coreos/bbolt/cmd/bolt/main.go b/vendor/github.com/coreos/bbolt/cmd/bolt/main.go new file mode 100644 index 0000000..b96e6f7 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/cmd/bolt/main.go @@ -0,0 +1,1532 @@ +package main + +import ( + "bytes" + "encoding/binary" + "errors" + "flag" + "fmt" + "io" + "io/ioutil" + "math/rand" + "os" + "runtime" + "runtime/pprof" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + "unsafe" + + "github.com/boltdb/bolt" +) + +var ( + // ErrUsage is returned when a usage message was printed and the process + // should simply exit with an error. + ErrUsage = errors.New("usage") + + // ErrUnknownCommand is returned when a CLI command is not specified. + ErrUnknownCommand = errors.New("unknown command") + + // ErrPathRequired is returned when the path to a Bolt database is not specified. + ErrPathRequired = errors.New("path required") + + // ErrFileNotFound is returned when a Bolt database does not exist. + ErrFileNotFound = errors.New("file not found") + + // ErrInvalidValue is returned when a benchmark reads an unexpected value. + ErrInvalidValue = errors.New("invalid value") + + // ErrCorrupt is returned when a checking a data file finds errors. + ErrCorrupt = errors.New("invalid value") + + // ErrNonDivisibleBatchSize is returned when the batch size can't be evenly + // divided by the iteration count. + ErrNonDivisibleBatchSize = errors.New("number of iterations must be divisible by the batch size") + + // ErrPageIDRequired is returned when a required page id is not specified. + ErrPageIDRequired = errors.New("page id required") + + // ErrPageNotFound is returned when specifying a page above the high water mark. + ErrPageNotFound = errors.New("page not found") + + // ErrPageFreed is returned when reading a page that has already been freed. + ErrPageFreed = errors.New("page freed") +) + +// PageHeaderSize represents the size of the bolt.page header. +const PageHeaderSize = 16 + +func main() { + m := NewMain() + if err := m.Run(os.Args[1:]...); err == ErrUsage { + os.Exit(2) + } else if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } +} + +// Main represents the main program execution. +type Main struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewMain returns a new instance of Main connect to the standard input/output. +func NewMain() *Main { + return &Main{ + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + } +} + +// Run executes the program. +func (m *Main) Run(args ...string) error { + // Require a command at the beginning. + if len(args) == 0 || strings.HasPrefix(args[0], "-") { + fmt.Fprintln(m.Stderr, m.Usage()) + return ErrUsage + } + + // Execute command. + switch args[0] { + case "help": + fmt.Fprintln(m.Stderr, m.Usage()) + return ErrUsage + case "bench": + return newBenchCommand(m).Run(args[1:]...) + case "check": + return newCheckCommand(m).Run(args[1:]...) + case "dump": + return newDumpCommand(m).Run(args[1:]...) + case "info": + return newInfoCommand(m).Run(args[1:]...) + case "page": + return newPageCommand(m).Run(args[1:]...) + case "pages": + return newPagesCommand(m).Run(args[1:]...) + case "stats": + return newStatsCommand(m).Run(args[1:]...) + default: + return ErrUnknownCommand + } +} + +// Usage returns the help message. +func (m *Main) Usage() string { + return strings.TrimLeft(` +Bolt is a tool for inspecting bolt databases. + +Usage: + + bolt command [arguments] + +The commands are: + + bench run synthetic benchmark against bolt + check verifies integrity of bolt database + info print basic info + help print this screen + pages print list of pages with their types + stats iterate over all pages and generate usage stats + +Use "bolt [command] -h" for more information about a command. +`, "\n") +} + +// CheckCommand represents the "check" command execution. +type CheckCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewCheckCommand returns a CheckCommand. +func newCheckCommand(m *Main) *CheckCommand { + return &CheckCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *CheckCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + help := fs.Bool("h", false, "") + if err := fs.Parse(args); err != nil { + return err + } else if *help { + fmt.Fprintln(cmd.Stderr, cmd.Usage()) + return ErrUsage + } + + // Require database path. + path := fs.Arg(0) + if path == "" { + return ErrPathRequired + } else if _, err := os.Stat(path); os.IsNotExist(err) { + return ErrFileNotFound + } + + // Open database. + db, err := bolt.Open(path, 0666, nil) + if err != nil { + return err + } + defer db.Close() + + // Perform consistency check. + return db.View(func(tx *bolt.Tx) error { + var count int + ch := tx.Check() + loop: + for { + select { + case err, ok := <-ch: + if !ok { + break loop + } + fmt.Fprintln(cmd.Stdout, err) + count++ + } + } + + // Print summary of errors. + if count > 0 { + fmt.Fprintf(cmd.Stdout, "%d errors found\n", count) + return ErrCorrupt + } + + // Notify user that database is valid. + fmt.Fprintln(cmd.Stdout, "OK") + return nil + }) +} + +// Usage returns the help message. +func (cmd *CheckCommand) Usage() string { + return strings.TrimLeft(` +usage: bolt check PATH + +Check opens a database at PATH and runs an exhaustive check to verify that +all pages are accessible or are marked as freed. It also verifies that no +pages are double referenced. + +Verification errors will stream out as they are found and the process will +return after all pages have been checked. +`, "\n") +} + +// InfoCommand represents the "info" command execution. +type InfoCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewInfoCommand returns a InfoCommand. +func newInfoCommand(m *Main) *InfoCommand { + return &InfoCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *InfoCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + help := fs.Bool("h", false, "") + if err := fs.Parse(args); err != nil { + return err + } else if *help { + fmt.Fprintln(cmd.Stderr, cmd.Usage()) + return ErrUsage + } + + // Require database path. + path := fs.Arg(0) + if path == "" { + return ErrPathRequired + } else if _, err := os.Stat(path); os.IsNotExist(err) { + return ErrFileNotFound + } + + // Open the database. + db, err := bolt.Open(path, 0666, nil) + if err != nil { + return err + } + defer db.Close() + + // Print basic database info. + info := db.Info() + fmt.Fprintf(cmd.Stdout, "Page Size: %d\n", info.PageSize) + + return nil +} + +// Usage returns the help message. +func (cmd *InfoCommand) Usage() string { + return strings.TrimLeft(` +usage: bolt info PATH + +Info prints basic information about the Bolt database at PATH. +`, "\n") +} + +// DumpCommand represents the "dump" command execution. +type DumpCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// newDumpCommand returns a DumpCommand. +func newDumpCommand(m *Main) *DumpCommand { + return &DumpCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *DumpCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + help := fs.Bool("h", false, "") + if err := fs.Parse(args); err != nil { + return err + } else if *help { + fmt.Fprintln(cmd.Stderr, cmd.Usage()) + return ErrUsage + } + + // Require database path and page id. + path := fs.Arg(0) + if path == "" { + return ErrPathRequired + } else if _, err := os.Stat(path); os.IsNotExist(err) { + return ErrFileNotFound + } + + // Read page ids. + pageIDs, err := atois(fs.Args()[1:]) + if err != nil { + return err + } else if len(pageIDs) == 0 { + return ErrPageIDRequired + } + + // Open database to retrieve page size. + pageSize, err := ReadPageSize(path) + if err != nil { + return err + } + + // Open database file handler. + f, err := os.Open(path) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + + // Print each page listed. + for i, pageID := range pageIDs { + // Print a separator. + if i > 0 { + fmt.Fprintln(cmd.Stdout, "===============================================") + } + + // Print page to stdout. + if err := cmd.PrintPage(cmd.Stdout, f, pageID, pageSize); err != nil { + return err + } + } + + return nil +} + +// PrintPage prints a given page as hexidecimal. +func (cmd *DumpCommand) PrintPage(w io.Writer, r io.ReaderAt, pageID int, pageSize int) error { + const bytesPerLineN = 16 + + // Read page into buffer. + buf := make([]byte, pageSize) + addr := pageID * pageSize + if n, err := r.ReadAt(buf, int64(addr)); err != nil { + return err + } else if n != pageSize { + return io.ErrUnexpectedEOF + } + + // Write out to writer in 16-byte lines. + var prev []byte + var skipped bool + for offset := 0; offset < pageSize; offset += bytesPerLineN { + // Retrieve current 16-byte line. + line := buf[offset : offset+bytesPerLineN] + isLastLine := (offset == (pageSize - bytesPerLineN)) + + // If it's the same as the previous line then print a skip. + if bytes.Equal(line, prev) && !isLastLine { + if !skipped { + fmt.Fprintf(w, "%07x *\n", addr+offset) + skipped = true + } + } else { + // Print line as hexadecimal in 2-byte groups. + fmt.Fprintf(w, "%07x %04x %04x %04x %04x %04x %04x %04x %04x\n", addr+offset, + line[0:2], line[2:4], line[4:6], line[6:8], + line[8:10], line[10:12], line[12:14], line[14:16], + ) + + skipped = false + } + + // Save the previous line. + prev = line + } + fmt.Fprint(w, "\n") + + return nil +} + +// Usage returns the help message. +func (cmd *DumpCommand) Usage() string { + return strings.TrimLeft(` +usage: bolt dump -page PAGEID PATH + +Dump prints a hexidecimal dump of a single page. +`, "\n") +} + +// PageCommand represents the "page" command execution. +type PageCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// newPageCommand returns a PageCommand. +func newPageCommand(m *Main) *PageCommand { + return &PageCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *PageCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + help := fs.Bool("h", false, "") + if err := fs.Parse(args); err != nil { + return err + } else if *help { + fmt.Fprintln(cmd.Stderr, cmd.Usage()) + return ErrUsage + } + + // Require database path and page id. + path := fs.Arg(0) + if path == "" { + return ErrPathRequired + } else if _, err := os.Stat(path); os.IsNotExist(err) { + return ErrFileNotFound + } + + // Read page ids. + pageIDs, err := atois(fs.Args()[1:]) + if err != nil { + return err + } else if len(pageIDs) == 0 { + return ErrPageIDRequired + } + + // Open database file handler. + f, err := os.Open(path) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + + // Print each page listed. + for i, pageID := range pageIDs { + // Print a separator. + if i > 0 { + fmt.Fprintln(cmd.Stdout, "===============================================") + } + + // Retrieve page info and page size. + p, buf, err := ReadPage(path, pageID) + if err != nil { + return err + } + + // Print basic page info. + fmt.Fprintf(cmd.Stdout, "Page ID: %d\n", p.id) + fmt.Fprintf(cmd.Stdout, "Page Type: %s\n", p.Type()) + fmt.Fprintf(cmd.Stdout, "Total Size: %d bytes\n", len(buf)) + + // Print type-specific data. + switch p.Type() { + case "meta": + err = cmd.PrintMeta(cmd.Stdout, buf) + case "leaf": + err = cmd.PrintLeaf(cmd.Stdout, buf) + case "branch": + err = cmd.PrintBranch(cmd.Stdout, buf) + case "freelist": + err = cmd.PrintFreelist(cmd.Stdout, buf) + } + if err != nil { + return err + } + } + + return nil +} + +// PrintMeta prints the data from the meta page. +func (cmd *PageCommand) PrintMeta(w io.Writer, buf []byte) error { + m := (*meta)(unsafe.Pointer(&buf[PageHeaderSize])) + fmt.Fprintf(w, "Version: %d\n", m.version) + fmt.Fprintf(w, "Page Size: %d bytes\n", m.pageSize) + fmt.Fprintf(w, "Flags: %08x\n", m.flags) + fmt.Fprintf(w, "Root: \n", m.root.root) + fmt.Fprintf(w, "Freelist: \n", m.freelist) + fmt.Fprintf(w, "HWM: \n", m.pgid) + fmt.Fprintf(w, "Txn ID: %d\n", m.txid) + fmt.Fprintf(w, "Checksum: %016x\n", m.checksum) + fmt.Fprintf(w, "\n") + return nil +} + +// PrintLeaf prints the data for a leaf page. +func (cmd *PageCommand) PrintLeaf(w io.Writer, buf []byte) error { + p := (*page)(unsafe.Pointer(&buf[0])) + + // Print number of items. + fmt.Fprintf(w, "Item Count: %d\n", p.count) + fmt.Fprintf(w, "\n") + + // Print each key/value. + for i := uint16(0); i < p.count; i++ { + e := p.leafPageElement(i) + + // Format key as string. + var k string + if isPrintable(string(e.key())) { + k = fmt.Sprintf("%q", string(e.key())) + } else { + k = fmt.Sprintf("%x", string(e.key())) + } + + // Format value as string. + var v string + if (e.flags & uint32(bucketLeafFlag)) != 0 { + b := (*bucket)(unsafe.Pointer(&e.value()[0])) + v = fmt.Sprintf("", b.root, b.sequence) + } else if isPrintable(string(e.value())) { + k = fmt.Sprintf("%q", string(e.value())) + } else { + k = fmt.Sprintf("%x", string(e.value())) + } + + fmt.Fprintf(w, "%s: %s\n", k, v) + } + fmt.Fprintf(w, "\n") + return nil +} + +// PrintBranch prints the data for a leaf page. +func (cmd *PageCommand) PrintBranch(w io.Writer, buf []byte) error { + p := (*page)(unsafe.Pointer(&buf[0])) + + // Print number of items. + fmt.Fprintf(w, "Item Count: %d\n", p.count) + fmt.Fprintf(w, "\n") + + // Print each key/value. + for i := uint16(0); i < p.count; i++ { + e := p.branchPageElement(i) + + // Format key as string. + var k string + if isPrintable(string(e.key())) { + k = fmt.Sprintf("%q", string(e.key())) + } else { + k = fmt.Sprintf("%x", string(e.key())) + } + + fmt.Fprintf(w, "%s: \n", k, e.pgid) + } + fmt.Fprintf(w, "\n") + return nil +} + +// PrintFreelist prints the data for a freelist page. +func (cmd *PageCommand) PrintFreelist(w io.Writer, buf []byte) error { + p := (*page)(unsafe.Pointer(&buf[0])) + + // Print number of items. + fmt.Fprintf(w, "Item Count: %d\n", p.count) + fmt.Fprintf(w, "\n") + + // Print each page in the freelist. + ids := (*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)) + for i := uint16(0); i < p.count; i++ { + fmt.Fprintf(w, "%d\n", ids[i]) + } + fmt.Fprintf(w, "\n") + return nil +} + +// PrintPage prints a given page as hexidecimal. +func (cmd *PageCommand) PrintPage(w io.Writer, r io.ReaderAt, pageID int, pageSize int) error { + const bytesPerLineN = 16 + + // Read page into buffer. + buf := make([]byte, pageSize) + addr := pageID * pageSize + if n, err := r.ReadAt(buf, int64(addr)); err != nil { + return err + } else if n != pageSize { + return io.ErrUnexpectedEOF + } + + // Write out to writer in 16-byte lines. + var prev []byte + var skipped bool + for offset := 0; offset < pageSize; offset += bytesPerLineN { + // Retrieve current 16-byte line. + line := buf[offset : offset+bytesPerLineN] + isLastLine := (offset == (pageSize - bytesPerLineN)) + + // If it's the same as the previous line then print a skip. + if bytes.Equal(line, prev) && !isLastLine { + if !skipped { + fmt.Fprintf(w, "%07x *\n", addr+offset) + skipped = true + } + } else { + // Print line as hexadecimal in 2-byte groups. + fmt.Fprintf(w, "%07x %04x %04x %04x %04x %04x %04x %04x %04x\n", addr+offset, + line[0:2], line[2:4], line[4:6], line[6:8], + line[8:10], line[10:12], line[12:14], line[14:16], + ) + + skipped = false + } + + // Save the previous line. + prev = line + } + fmt.Fprint(w, "\n") + + return nil +} + +// Usage returns the help message. +func (cmd *PageCommand) Usage() string { + return strings.TrimLeft(` +usage: bolt page -page PATH pageid [pageid...] + +Page prints one or more pages in human readable format. +`, "\n") +} + +// PagesCommand represents the "pages" command execution. +type PagesCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewPagesCommand returns a PagesCommand. +func newPagesCommand(m *Main) *PagesCommand { + return &PagesCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *PagesCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + help := fs.Bool("h", false, "") + if err := fs.Parse(args); err != nil { + return err + } else if *help { + fmt.Fprintln(cmd.Stderr, cmd.Usage()) + return ErrUsage + } + + // Require database path. + path := fs.Arg(0) + if path == "" { + return ErrPathRequired + } else if _, err := os.Stat(path); os.IsNotExist(err) { + return ErrFileNotFound + } + + // Open database. + db, err := bolt.Open(path, 0666, nil) + if err != nil { + return err + } + defer func() { _ = db.Close() }() + + // Write header. + fmt.Fprintln(cmd.Stdout, "ID TYPE ITEMS OVRFLW") + fmt.Fprintln(cmd.Stdout, "======== ========== ====== ======") + + return db.Update(func(tx *bolt.Tx) error { + var id int + for { + p, err := tx.Page(id) + if err != nil { + return &PageError{ID: id, Err: err} + } else if p == nil { + break + } + + // Only display count and overflow if this is a non-free page. + var count, overflow string + if p.Type != "free" { + count = strconv.Itoa(p.Count) + if p.OverflowCount > 0 { + overflow = strconv.Itoa(p.OverflowCount) + } + } + + // Print table row. + fmt.Fprintf(cmd.Stdout, "%-8d %-10s %-6s %-6s\n", p.ID, p.Type, count, overflow) + + // Move to the next non-overflow page. + id += 1 + if p.Type != "free" { + id += p.OverflowCount + } + } + return nil + }) +} + +// Usage returns the help message. +func (cmd *PagesCommand) Usage() string { + return strings.TrimLeft(` +usage: bolt pages PATH + +Pages prints a table of pages with their type (meta, leaf, branch, freelist). +Leaf and branch pages will show a key count in the "items" column while the +freelist will show the number of free pages in the "items" column. + +The "overflow" column shows the number of blocks that the page spills over +into. Normally there is no overflow but large keys and values can cause +a single page to take up multiple blocks. +`, "\n") +} + +// StatsCommand represents the "stats" command execution. +type StatsCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewStatsCommand returns a StatsCommand. +func newStatsCommand(m *Main) *StatsCommand { + return &StatsCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *StatsCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + help := fs.Bool("h", false, "") + if err := fs.Parse(args); err != nil { + return err + } else if *help { + fmt.Fprintln(cmd.Stderr, cmd.Usage()) + return ErrUsage + } + + // Require database path. + path, prefix := fs.Arg(0), fs.Arg(1) + if path == "" { + return ErrPathRequired + } else if _, err := os.Stat(path); os.IsNotExist(err) { + return ErrFileNotFound + } + + // Open database. + db, err := bolt.Open(path, 0666, nil) + if err != nil { + return err + } + defer db.Close() + + return db.View(func(tx *bolt.Tx) error { + var s bolt.BucketStats + var count int + if err := tx.ForEach(func(name []byte, b *bolt.Bucket) error { + if bytes.HasPrefix(name, []byte(prefix)) { + s.Add(b.Stats()) + count += 1 + } + return nil + }); err != nil { + return err + } + + fmt.Fprintf(cmd.Stdout, "Aggregate statistics for %d buckets\n\n", count) + + fmt.Fprintln(cmd.Stdout, "Page count statistics") + fmt.Fprintf(cmd.Stdout, "\tNumber of logical branch pages: %d\n", s.BranchPageN) + fmt.Fprintf(cmd.Stdout, "\tNumber of physical branch overflow pages: %d\n", s.BranchOverflowN) + fmt.Fprintf(cmd.Stdout, "\tNumber of logical leaf pages: %d\n", s.LeafPageN) + fmt.Fprintf(cmd.Stdout, "\tNumber of physical leaf overflow pages: %d\n", s.LeafOverflowN) + + fmt.Fprintln(cmd.Stdout, "Tree statistics") + fmt.Fprintf(cmd.Stdout, "\tNumber of keys/value pairs: %d\n", s.KeyN) + fmt.Fprintf(cmd.Stdout, "\tNumber of levels in B+tree: %d\n", s.Depth) + + fmt.Fprintln(cmd.Stdout, "Page size utilization") + fmt.Fprintf(cmd.Stdout, "\tBytes allocated for physical branch pages: %d\n", s.BranchAlloc) + var percentage int + if s.BranchAlloc != 0 { + percentage = int(float32(s.BranchInuse) * 100.0 / float32(s.BranchAlloc)) + } + fmt.Fprintf(cmd.Stdout, "\tBytes actually used for branch data: %d (%d%%)\n", s.BranchInuse, percentage) + fmt.Fprintf(cmd.Stdout, "\tBytes allocated for physical leaf pages: %d\n", s.LeafAlloc) + percentage = 0 + if s.LeafAlloc != 0 { + percentage = int(float32(s.LeafInuse) * 100.0 / float32(s.LeafAlloc)) + } + fmt.Fprintf(cmd.Stdout, "\tBytes actually used for leaf data: %d (%d%%)\n", s.LeafInuse, percentage) + + fmt.Fprintln(cmd.Stdout, "Bucket statistics") + fmt.Fprintf(cmd.Stdout, "\tTotal number of buckets: %d\n", s.BucketN) + percentage = 0 + if s.BucketN != 0 { + percentage = int(float32(s.InlineBucketN) * 100.0 / float32(s.BucketN)) + } + fmt.Fprintf(cmd.Stdout, "\tTotal number on inlined buckets: %d (%d%%)\n", s.InlineBucketN, percentage) + percentage = 0 + if s.LeafInuse != 0 { + percentage = int(float32(s.InlineBucketInuse) * 100.0 / float32(s.LeafInuse)) + } + fmt.Fprintf(cmd.Stdout, "\tBytes used for inlined buckets: %d (%d%%)\n", s.InlineBucketInuse, percentage) + + return nil + }) +} + +// Usage returns the help message. +func (cmd *StatsCommand) Usage() string { + return strings.TrimLeft(` +usage: bolt stats PATH + +Stats performs an extensive search of the database to track every page +reference. It starts at the current meta page and recursively iterates +through every accessible bucket. + +The following errors can be reported: + + already freed + The page is referenced more than once in the freelist. + + unreachable unfreed + The page is not referenced by a bucket or in the freelist. + + reachable freed + The page is referenced by a bucket but is also in the freelist. + + out of bounds + A page is referenced that is above the high water mark. + + multiple references + A page is referenced by more than one other page. + + invalid type + The page type is not "meta", "leaf", "branch", or "freelist". + +No errors should occur in your database. However, if for some reason you +experience corruption, please submit a ticket to the Bolt project page: + + https://github.com/boltdb/bolt/issues +`, "\n") +} + +var benchBucketName = []byte("bench") + +// BenchCommand represents the "bench" command execution. +type BenchCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewBenchCommand returns a BenchCommand using the +func newBenchCommand(m *Main) *BenchCommand { + return &BenchCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the "bench" command. +func (cmd *BenchCommand) Run(args ...string) error { + // Parse CLI arguments. + options, err := cmd.ParseFlags(args) + if err != nil { + return err + } + + // Remove path if "-work" is not set. Otherwise keep path. + if options.Work { + fmt.Fprintf(cmd.Stdout, "work: %s\n", options.Path) + } else { + defer os.Remove(options.Path) + } + + // Create database. + db, err := bolt.Open(options.Path, 0666, nil) + if err != nil { + return err + } + db.NoSync = options.NoSync + defer db.Close() + + // Write to the database. + var results BenchResults + if err := cmd.runWrites(db, options, &results); err != nil { + return fmt.Errorf("write: %v", err) + } + + // Read from the database. + if err := cmd.runReads(db, options, &results); err != nil { + return fmt.Errorf("bench: read: %s", err) + } + + // Print results. + fmt.Fprintf(os.Stderr, "# Write\t%v\t(%v/op)\t(%v op/sec)\n", results.WriteDuration, results.WriteOpDuration(), results.WriteOpsPerSecond()) + fmt.Fprintf(os.Stderr, "# Read\t%v\t(%v/op)\t(%v op/sec)\n", results.ReadDuration, results.ReadOpDuration(), results.ReadOpsPerSecond()) + fmt.Fprintln(os.Stderr, "") + return nil +} + +// ParseFlags parses the command line flags. +func (cmd *BenchCommand) ParseFlags(args []string) (*BenchOptions, error) { + var options BenchOptions + + // Parse flagset. + fs := flag.NewFlagSet("", flag.ContinueOnError) + fs.StringVar(&options.ProfileMode, "profile-mode", "rw", "") + fs.StringVar(&options.WriteMode, "write-mode", "seq", "") + fs.StringVar(&options.ReadMode, "read-mode", "seq", "") + fs.IntVar(&options.Iterations, "count", 1000, "") + fs.IntVar(&options.BatchSize, "batch-size", 0, "") + fs.IntVar(&options.KeySize, "key-size", 8, "") + fs.IntVar(&options.ValueSize, "value-size", 32, "") + fs.StringVar(&options.CPUProfile, "cpuprofile", "", "") + fs.StringVar(&options.MemProfile, "memprofile", "", "") + fs.StringVar(&options.BlockProfile, "blockprofile", "", "") + fs.Float64Var(&options.FillPercent, "fill-percent", bolt.DefaultFillPercent, "") + fs.BoolVar(&options.NoSync, "no-sync", false, "") + fs.BoolVar(&options.Work, "work", false, "") + fs.StringVar(&options.Path, "path", "", "") + fs.SetOutput(cmd.Stderr) + if err := fs.Parse(args); err != nil { + return nil, err + } + + // Set batch size to iteration size if not set. + // Require that batch size can be evenly divided by the iteration count. + if options.BatchSize == 0 { + options.BatchSize = options.Iterations + } else if options.Iterations%options.BatchSize != 0 { + return nil, ErrNonDivisibleBatchSize + } + + // Generate temp path if one is not passed in. + if options.Path == "" { + f, err := ioutil.TempFile("", "bolt-bench-") + if err != nil { + return nil, fmt.Errorf("temp file: %s", err) + } + f.Close() + os.Remove(f.Name()) + options.Path = f.Name() + } + + return &options, nil +} + +// Writes to the database. +func (cmd *BenchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + // Start profiling for writes. + if options.ProfileMode == "rw" || options.ProfileMode == "w" { + cmd.startProfiling(options) + } + + t := time.Now() + + var err error + switch options.WriteMode { + case "seq": + err = cmd.runWritesSequential(db, options, results) + case "rnd": + err = cmd.runWritesRandom(db, options, results) + case "seq-nest": + err = cmd.runWritesSequentialNested(db, options, results) + case "rnd-nest": + err = cmd.runWritesRandomNested(db, options, results) + default: + return fmt.Errorf("invalid write mode: %s", options.WriteMode) + } + + // Save time to write. + results.WriteDuration = time.Since(t) + + // Stop profiling for writes only. + if options.ProfileMode == "w" { + cmd.stopProfiling() + } + + return err +} + +func (cmd *BenchCommand) runWritesSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + var i = uint32(0) + return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i }) +} + +func (cmd *BenchCommand) runWritesRandom(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() }) +} + +func (cmd *BenchCommand) runWritesSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + var i = uint32(0) + return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i }) +} + +func (cmd *BenchCommand) runWritesRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() }) +} + +func (cmd *BenchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error { + results.WriteOps = options.Iterations + + for i := 0; i < options.Iterations; i += options.BatchSize { + if err := db.Update(func(tx *bolt.Tx) error { + b, _ := tx.CreateBucketIfNotExists(benchBucketName) + b.FillPercent = options.FillPercent + + for j := 0; j < options.BatchSize; j++ { + key := make([]byte, options.KeySize) + value := make([]byte, options.ValueSize) + + // Write key as uint32. + binary.BigEndian.PutUint32(key, keySource()) + + // Insert key/value. + if err := b.Put(key, value); err != nil { + return err + } + } + + return nil + }); err != nil { + return err + } + } + return nil +} + +func (cmd *BenchCommand) runWritesNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error { + results.WriteOps = options.Iterations + + for i := 0; i < options.Iterations; i += options.BatchSize { + if err := db.Update(func(tx *bolt.Tx) error { + top, err := tx.CreateBucketIfNotExists(benchBucketName) + if err != nil { + return err + } + top.FillPercent = options.FillPercent + + // Create bucket key. + name := make([]byte, options.KeySize) + binary.BigEndian.PutUint32(name, keySource()) + + // Create bucket. + b, err := top.CreateBucketIfNotExists(name) + if err != nil { + return err + } + b.FillPercent = options.FillPercent + + for j := 0; j < options.BatchSize; j++ { + var key = make([]byte, options.KeySize) + var value = make([]byte, options.ValueSize) + + // Generate key as uint32. + binary.BigEndian.PutUint32(key, keySource()) + + // Insert value into subbucket. + if err := b.Put(key, value); err != nil { + return err + } + } + + return nil + }); err != nil { + return err + } + } + return nil +} + +// Reads from the database. +func (cmd *BenchCommand) runReads(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + // Start profiling for reads. + if options.ProfileMode == "r" { + cmd.startProfiling(options) + } + + t := time.Now() + + var err error + switch options.ReadMode { + case "seq": + switch options.WriteMode { + case "seq-nest", "rnd-nest": + err = cmd.runReadsSequentialNested(db, options, results) + default: + err = cmd.runReadsSequential(db, options, results) + } + default: + return fmt.Errorf("invalid read mode: %s", options.ReadMode) + } + + // Save read time. + results.ReadDuration = time.Since(t) + + // Stop profiling for reads. + if options.ProfileMode == "rw" || options.ProfileMode == "r" { + cmd.stopProfiling() + } + + return err +} + +func (cmd *BenchCommand) runReadsSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + return db.View(func(tx *bolt.Tx) error { + t := time.Now() + + for { + var count int + + c := tx.Bucket(benchBucketName).Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + if v == nil { + return errors.New("invalid value") + } + count++ + } + + if options.WriteMode == "seq" && count != options.Iterations { + return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, count) + } + + results.ReadOps += count + + // Make sure we do this for at least a second. + if time.Since(t) >= time.Second { + break + } + } + + return nil + }) +} + +func (cmd *BenchCommand) runReadsSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + return db.View(func(tx *bolt.Tx) error { + t := time.Now() + + for { + var count int + var top = tx.Bucket(benchBucketName) + if err := top.ForEach(func(name, _ []byte) error { + c := top.Bucket(name).Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + if v == nil { + return ErrInvalidValue + } + count++ + } + return nil + }); err != nil { + return err + } + + if options.WriteMode == "seq-nest" && count != options.Iterations { + return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, count) + } + + results.ReadOps += count + + // Make sure we do this for at least a second. + if time.Since(t) >= time.Second { + break + } + } + + return nil + }) +} + +// File handlers for the various profiles. +var cpuprofile, memprofile, blockprofile *os.File + +// Starts all profiles set on the options. +func (cmd *BenchCommand) startProfiling(options *BenchOptions) { + var err error + + // Start CPU profiling. + if options.CPUProfile != "" { + cpuprofile, err = os.Create(options.CPUProfile) + if err != nil { + fmt.Fprintf(cmd.Stderr, "bench: could not create cpu profile %q: %v\n", options.CPUProfile, err) + os.Exit(1) + } + pprof.StartCPUProfile(cpuprofile) + } + + // Start memory profiling. + if options.MemProfile != "" { + memprofile, err = os.Create(options.MemProfile) + if err != nil { + fmt.Fprintf(cmd.Stderr, "bench: could not create memory profile %q: %v\n", options.MemProfile, err) + os.Exit(1) + } + runtime.MemProfileRate = 4096 + } + + // Start fatal profiling. + if options.BlockProfile != "" { + blockprofile, err = os.Create(options.BlockProfile) + if err != nil { + fmt.Fprintf(cmd.Stderr, "bench: could not create block profile %q: %v\n", options.BlockProfile, err) + os.Exit(1) + } + runtime.SetBlockProfileRate(1) + } +} + +// Stops all profiles. +func (cmd *BenchCommand) stopProfiling() { + if cpuprofile != nil { + pprof.StopCPUProfile() + cpuprofile.Close() + cpuprofile = nil + } + + if memprofile != nil { + pprof.Lookup("heap").WriteTo(memprofile, 0) + memprofile.Close() + memprofile = nil + } + + if blockprofile != nil { + pprof.Lookup("block").WriteTo(blockprofile, 0) + blockprofile.Close() + blockprofile = nil + runtime.SetBlockProfileRate(0) + } +} + +// BenchOptions represents the set of options that can be passed to "bolt bench". +type BenchOptions struct { + ProfileMode string + WriteMode string + ReadMode string + Iterations int + BatchSize int + KeySize int + ValueSize int + CPUProfile string + MemProfile string + BlockProfile string + StatsInterval time.Duration + FillPercent float64 + NoSync bool + Work bool + Path string +} + +// BenchResults represents the performance results of the benchmark. +type BenchResults struct { + WriteOps int + WriteDuration time.Duration + ReadOps int + ReadDuration time.Duration +} + +// Returns the duration for a single write operation. +func (r *BenchResults) WriteOpDuration() time.Duration { + if r.WriteOps == 0 { + return 0 + } + return r.WriteDuration / time.Duration(r.WriteOps) +} + +// Returns average number of write operations that can be performed per second. +func (r *BenchResults) WriteOpsPerSecond() int { + var op = r.WriteOpDuration() + if op == 0 { + return 0 + } + return int(time.Second) / int(op) +} + +// Returns the duration for a single read operation. +func (r *BenchResults) ReadOpDuration() time.Duration { + if r.ReadOps == 0 { + return 0 + } + return r.ReadDuration / time.Duration(r.ReadOps) +} + +// Returns average number of read operations that can be performed per second. +func (r *BenchResults) ReadOpsPerSecond() int { + var op = r.ReadOpDuration() + if op == 0 { + return 0 + } + return int(time.Second) / int(op) +} + +type PageError struct { + ID int + Err error +} + +func (e *PageError) Error() string { + return fmt.Sprintf("page error: id=%d, err=%s", e.ID, e.Err) +} + +// isPrintable returns true if the string is valid unicode and contains only printable runes. +func isPrintable(s string) bool { + if !utf8.ValidString(s) { + return false + } + for _, ch := range s { + if !unicode.IsPrint(ch) { + return false + } + } + return true +} + +// ReadPage reads page info & full page data from a path. +// This is not transactionally safe. +func ReadPage(path string, pageID int) (*page, []byte, error) { + // Find page size. + pageSize, err := ReadPageSize(path) + if err != nil { + return nil, nil, fmt.Errorf("read page size: %s", err) + } + + // Open database file. + f, err := os.Open(path) + if err != nil { + return nil, nil, err + } + defer f.Close() + + // Read one block into buffer. + buf := make([]byte, pageSize) + if n, err := f.ReadAt(buf, int64(pageID*pageSize)); err != nil { + return nil, nil, err + } else if n != len(buf) { + return nil, nil, io.ErrUnexpectedEOF + } + + // Determine total number of blocks. + p := (*page)(unsafe.Pointer(&buf[0])) + overflowN := p.overflow + + // Re-read entire page (with overflow) into buffer. + buf = make([]byte, (int(overflowN)+1)*pageSize) + if n, err := f.ReadAt(buf, int64(pageID*pageSize)); err != nil { + return nil, nil, err + } else if n != len(buf) { + return nil, nil, io.ErrUnexpectedEOF + } + p = (*page)(unsafe.Pointer(&buf[0])) + + return p, buf, nil +} + +// ReadPageSize reads page size a path. +// This is not transactionally safe. +func ReadPageSize(path string) (int, error) { + // Open database file. + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + + // Read 4KB chunk. + buf := make([]byte, 4096) + if _, err := io.ReadFull(f, buf); err != nil { + return 0, err + } + + // Read page size from metadata. + m := (*meta)(unsafe.Pointer(&buf[PageHeaderSize])) + return int(m.pageSize), nil +} + +// atois parses a slice of strings into integers. +func atois(strs []string) ([]int, error) { + var a []int + for _, str := range strs { + i, err := strconv.Atoi(str) + if err != nil { + return nil, err + } + a = append(a, i) + } + return a, nil +} + +// DO NOT EDIT. Copied from the "bolt" package. +const maxAllocSize = 0xFFFFFFF + +// DO NOT EDIT. Copied from the "bolt" package. +const ( + branchPageFlag = 0x01 + leafPageFlag = 0x02 + metaPageFlag = 0x04 + freelistPageFlag = 0x10 +) + +// DO NOT EDIT. Copied from the "bolt" package. +const bucketLeafFlag = 0x01 + +// DO NOT EDIT. Copied from the "bolt" package. +type pgid uint64 + +// DO NOT EDIT. Copied from the "bolt" package. +type txid uint64 + +// DO NOT EDIT. Copied from the "bolt" package. +type meta struct { + magic uint32 + version uint32 + pageSize uint32 + flags uint32 + root bucket + freelist pgid + pgid pgid + txid txid + checksum uint64 +} + +// DO NOT EDIT. Copied from the "bolt" package. +type bucket struct { + root pgid + sequence uint64 +} + +// DO NOT EDIT. Copied from the "bolt" package. +type page struct { + id pgid + flags uint16 + count uint16 + overflow uint32 + ptr uintptr +} + +// DO NOT EDIT. Copied from the "bolt" package. +func (p *page) Type() string { + if (p.flags & branchPageFlag) != 0 { + return "branch" + } else if (p.flags & leafPageFlag) != 0 { + return "leaf" + } else if (p.flags & metaPageFlag) != 0 { + return "meta" + } else if (p.flags & freelistPageFlag) != 0 { + return "freelist" + } + return fmt.Sprintf("unknown<%02x>", p.flags) +} + +// DO NOT EDIT. Copied from the "bolt" package. +func (p *page) leafPageElement(index uint16) *leafPageElement { + n := &((*[0x7FFFFFF]leafPageElement)(unsafe.Pointer(&p.ptr)))[index] + return n +} + +// DO NOT EDIT. Copied from the "bolt" package. +func (p *page) branchPageElement(index uint16) *branchPageElement { + return &((*[0x7FFFFFF]branchPageElement)(unsafe.Pointer(&p.ptr)))[index] +} + +// DO NOT EDIT. Copied from the "bolt" package. +type branchPageElement struct { + pos uint32 + ksize uint32 + pgid pgid +} + +// DO NOT EDIT. Copied from the "bolt" package. +func (n *branchPageElement) key() []byte { + buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) + return buf[n.pos : n.pos+n.ksize] +} + +// DO NOT EDIT. Copied from the "bolt" package. +type leafPageElement struct { + flags uint32 + pos uint32 + ksize uint32 + vsize uint32 +} + +// DO NOT EDIT. Copied from the "bolt" package. +func (n *leafPageElement) key() []byte { + buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) + return buf[n.pos : n.pos+n.ksize] +} + +// DO NOT EDIT. Copied from the "bolt" package. +func (n *leafPageElement) value() []byte { + buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) + return buf[n.pos+n.ksize : n.pos+n.ksize+n.vsize] +} diff --git a/vendor/github.com/coreos/bbolt/cmd/bolt/main_test.go b/vendor/github.com/coreos/bbolt/cmd/bolt/main_test.go new file mode 100644 index 0000000..c378b79 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/cmd/bolt/main_test.go @@ -0,0 +1,185 @@ +package main_test + +import ( + "bytes" + "io/ioutil" + "os" + "strconv" + "testing" + + "github.com/boltdb/bolt" + "github.com/boltdb/bolt/cmd/bolt" +) + +// Ensure the "info" command can print information about a database. +func TestInfoCommand_Run(t *testing.T) { + db := MustOpen(0666, nil) + db.DB.Close() + defer db.Close() + + // Run the info command. + m := NewMain() + if err := m.Run("info", db.Path); err != nil { + t.Fatal(err) + } +} + +// Ensure the "stats" command executes correctly with an empty database. +func TestStatsCommand_Run_EmptyDatabase(t *testing.T) { + // Ignore + if os.Getpagesize() != 4096 { + t.Skip("system does not use 4KB page size") + } + + db := MustOpen(0666, nil) + defer db.Close() + db.DB.Close() + + // Generate expected result. + exp := "Aggregate statistics for 0 buckets\n\n" + + "Page count statistics\n" + + "\tNumber of logical branch pages: 0\n" + + "\tNumber of physical branch overflow pages: 0\n" + + "\tNumber of logical leaf pages: 0\n" + + "\tNumber of physical leaf overflow pages: 0\n" + + "Tree statistics\n" + + "\tNumber of keys/value pairs: 0\n" + + "\tNumber of levels in B+tree: 0\n" + + "Page size utilization\n" + + "\tBytes allocated for physical branch pages: 0\n" + + "\tBytes actually used for branch data: 0 (0%)\n" + + "\tBytes allocated for physical leaf pages: 0\n" + + "\tBytes actually used for leaf data: 0 (0%)\n" + + "Bucket statistics\n" + + "\tTotal number of buckets: 0\n" + + "\tTotal number on inlined buckets: 0 (0%)\n" + + "\tBytes used for inlined buckets: 0 (0%)\n" + + // Run the command. + m := NewMain() + if err := m.Run("stats", db.Path); err != nil { + t.Fatal(err) + } else if m.Stdout.String() != exp { + t.Fatalf("unexpected stdout:\n\n%s", m.Stdout.String()) + } +} + +// Ensure the "stats" command can execute correctly. +func TestStatsCommand_Run(t *testing.T) { + // Ignore + if os.Getpagesize() != 4096 { + t.Skip("system does not use 4KB page size") + } + + db := MustOpen(0666, nil) + defer db.Close() + + if err := db.Update(func(tx *bolt.Tx) error { + // Create "foo" bucket. + b, err := tx.CreateBucket([]byte("foo")) + if err != nil { + return err + } + for i := 0; i < 10; i++ { + if err := b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil { + return err + } + } + + // Create "bar" bucket. + b, err = tx.CreateBucket([]byte("bar")) + if err != nil { + return err + } + for i := 0; i < 100; i++ { + if err := b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil { + return err + } + } + + // Create "baz" bucket. + b, err = tx.CreateBucket([]byte("baz")) + if err != nil { + return err + } + if err := b.Put([]byte("key"), []byte("value")); err != nil { + return err + } + + return nil + }); err != nil { + t.Fatal(err) + } + db.DB.Close() + + // Generate expected result. + exp := "Aggregate statistics for 3 buckets\n\n" + + "Page count statistics\n" + + "\tNumber of logical branch pages: 0\n" + + "\tNumber of physical branch overflow pages: 0\n" + + "\tNumber of logical leaf pages: 1\n" + + "\tNumber of physical leaf overflow pages: 0\n" + + "Tree statistics\n" + + "\tNumber of keys/value pairs: 111\n" + + "\tNumber of levels in B+tree: 1\n" + + "Page size utilization\n" + + "\tBytes allocated for physical branch pages: 0\n" + + "\tBytes actually used for branch data: 0 (0%)\n" + + "\tBytes allocated for physical leaf pages: 4096\n" + + "\tBytes actually used for leaf data: 1996 (48%)\n" + + "Bucket statistics\n" + + "\tTotal number of buckets: 3\n" + + "\tTotal number on inlined buckets: 2 (66%)\n" + + "\tBytes used for inlined buckets: 236 (11%)\n" + + // Run the command. + m := NewMain() + if err := m.Run("stats", db.Path); err != nil { + t.Fatal(err) + } else if m.Stdout.String() != exp { + t.Fatalf("unexpected stdout:\n\n%s", m.Stdout.String()) + } +} + +// Main represents a test wrapper for main.Main that records output. +type Main struct { + *main.Main + Stdin bytes.Buffer + Stdout bytes.Buffer + Stderr bytes.Buffer +} + +// NewMain returns a new instance of Main. +func NewMain() *Main { + m := &Main{Main: main.NewMain()} + m.Main.Stdin = &m.Stdin + m.Main.Stdout = &m.Stdout + m.Main.Stderr = &m.Stderr + return m +} + +// MustOpen creates a Bolt database in a temporary location. +func MustOpen(mode os.FileMode, options *bolt.Options) *DB { + // Create temporary path. + f, _ := ioutil.TempFile("", "bolt-") + f.Close() + os.Remove(f.Name()) + + db, err := bolt.Open(f.Name(), mode, options) + if err != nil { + panic(err.Error()) + } + return &DB{DB: db, Path: f.Name()} +} + +// DB is a test wrapper for bolt.DB. +type DB struct { + *bolt.DB + Path string +} + +// Close closes and removes the database. +func (db *DB) Close() error { + defer os.Remove(db.Path) + return db.DB.Close() +} diff --git a/vendor/github.com/coreos/bbolt/cursor.go b/vendor/github.com/coreos/bbolt/cursor.go new file mode 100644 index 0000000..1be9f35 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/cursor.go @@ -0,0 +1,400 @@ +package bolt + +import ( + "bytes" + "fmt" + "sort" +) + +// Cursor represents an iterator that can traverse over all key/value pairs in a bucket in sorted order. +// Cursors see nested buckets with value == nil. +// Cursors can be obtained from a transaction and are valid as long as the transaction is open. +// +// Keys and values returned from the cursor are only valid for the life of the transaction. +// +// Changing data while traversing with a cursor may cause it to be invalidated +// and return unexpected keys and/or values. You must reposition your cursor +// after mutating data. +type Cursor struct { + bucket *Bucket + stack []elemRef +} + +// Bucket returns the bucket that this cursor was created from. +func (c *Cursor) Bucket() *Bucket { + return c.bucket +} + +// First moves the cursor to the first item in the bucket and returns its key and value. +// If the bucket is empty then a nil key and value are returned. +// The returned key and value are only valid for the life of the transaction. +func (c *Cursor) First() (key []byte, value []byte) { + _assert(c.bucket.tx.db != nil, "tx closed") + c.stack = c.stack[:0] + p, n := c.bucket.pageNode(c.bucket.root) + c.stack = append(c.stack, elemRef{page: p, node: n, index: 0}) + c.first() + + // If we land on an empty page then move to the next value. + // https://github.com/boltdb/bolt/issues/450 + if c.stack[len(c.stack)-1].count() == 0 { + c.next() + } + + k, v, flags := c.keyValue() + if (flags & uint32(bucketLeafFlag)) != 0 { + return k, nil + } + return k, v + +} + +// Last moves the cursor to the last item in the bucket and returns its key and value. +// If the bucket is empty then a nil key and value are returned. +// The returned key and value are only valid for the life of the transaction. +func (c *Cursor) Last() (key []byte, value []byte) { + _assert(c.bucket.tx.db != nil, "tx closed") + c.stack = c.stack[:0] + p, n := c.bucket.pageNode(c.bucket.root) + ref := elemRef{page: p, node: n} + ref.index = ref.count() - 1 + c.stack = append(c.stack, ref) + c.last() + k, v, flags := c.keyValue() + if (flags & uint32(bucketLeafFlag)) != 0 { + return k, nil + } + return k, v +} + +// Next moves the cursor to the next item in the bucket and returns its key and value. +// If the cursor is at the end of the bucket then a nil key and value are returned. +// The returned key and value are only valid for the life of the transaction. +func (c *Cursor) Next() (key []byte, value []byte) { + _assert(c.bucket.tx.db != nil, "tx closed") + k, v, flags := c.next() + if (flags & uint32(bucketLeafFlag)) != 0 { + return k, nil + } + return k, v +} + +// Prev moves the cursor to the previous item in the bucket and returns its key and value. +// If the cursor is at the beginning of the bucket then a nil key and value are returned. +// The returned key and value are only valid for the life of the transaction. +func (c *Cursor) Prev() (key []byte, value []byte) { + _assert(c.bucket.tx.db != nil, "tx closed") + + // Attempt to move back one element until we're successful. + // Move up the stack as we hit the beginning of each page in our stack. + for i := len(c.stack) - 1; i >= 0; i-- { + elem := &c.stack[i] + if elem.index > 0 { + elem.index-- + break + } + c.stack = c.stack[:i] + } + + // If we've hit the end then return nil. + if len(c.stack) == 0 { + return nil, nil + } + + // Move down the stack to find the last element of the last leaf under this branch. + c.last() + k, v, flags := c.keyValue() + if (flags & uint32(bucketLeafFlag)) != 0 { + return k, nil + } + return k, v +} + +// Seek moves the cursor to a given key and returns it. +// If the key does not exist then the next key is used. If no keys +// follow, a nil key is returned. +// The returned key and value are only valid for the life of the transaction. +func (c *Cursor) Seek(seek []byte) (key []byte, value []byte) { + k, v, flags := c.seek(seek) + + // If we ended up after the last element of a page then move to the next one. + if ref := &c.stack[len(c.stack)-1]; ref.index >= ref.count() { + k, v, flags = c.next() + } + + if k == nil { + return nil, nil + } else if (flags & uint32(bucketLeafFlag)) != 0 { + return k, nil + } + return k, v +} + +// Delete removes the current key/value under the cursor from the bucket. +// Delete fails if current key/value is a bucket or if the transaction is not writable. +func (c *Cursor) Delete() error { + if c.bucket.tx.db == nil { + return ErrTxClosed + } else if !c.bucket.Writable() { + return ErrTxNotWritable + } + + key, _, flags := c.keyValue() + // Return an error if current value is a bucket. + if (flags & bucketLeafFlag) != 0 { + return ErrIncompatibleValue + } + c.node().del(key) + + return nil +} + +// seek moves the cursor to a given key and returns it. +// If the key does not exist then the next key is used. +func (c *Cursor) seek(seek []byte) (key []byte, value []byte, flags uint32) { + _assert(c.bucket.tx.db != nil, "tx closed") + + // Start from root page/node and traverse to correct page. + c.stack = c.stack[:0] + c.search(seek, c.bucket.root) + ref := &c.stack[len(c.stack)-1] + + // If the cursor is pointing to the end of page/node then return nil. + if ref.index >= ref.count() { + return nil, nil, 0 + } + + // If this is a bucket then return a nil value. + return c.keyValue() +} + +// first moves the cursor to the first leaf element under the last page in the stack. +func (c *Cursor) first() { + for { + // Exit when we hit a leaf page. + var ref = &c.stack[len(c.stack)-1] + if ref.isLeaf() { + break + } + + // Keep adding pages pointing to the first element to the stack. + var pgid pgid + if ref.node != nil { + pgid = ref.node.inodes[ref.index].pgid + } else { + pgid = ref.page.branchPageElement(uint16(ref.index)).pgid + } + p, n := c.bucket.pageNode(pgid) + c.stack = append(c.stack, elemRef{page: p, node: n, index: 0}) + } +} + +// last moves the cursor to the last leaf element under the last page in the stack. +func (c *Cursor) last() { + for { + // Exit when we hit a leaf page. + ref := &c.stack[len(c.stack)-1] + if ref.isLeaf() { + break + } + + // Keep adding pages pointing to the last element in the stack. + var pgid pgid + if ref.node != nil { + pgid = ref.node.inodes[ref.index].pgid + } else { + pgid = ref.page.branchPageElement(uint16(ref.index)).pgid + } + p, n := c.bucket.pageNode(pgid) + + var nextRef = elemRef{page: p, node: n} + nextRef.index = nextRef.count() - 1 + c.stack = append(c.stack, nextRef) + } +} + +// next moves to the next leaf element and returns the key and value. +// If the cursor is at the last leaf element then it stays there and returns nil. +func (c *Cursor) next() (key []byte, value []byte, flags uint32) { + for { + // Attempt to move over one element until we're successful. + // Move up the stack as we hit the end of each page in our stack. + var i int + for i = len(c.stack) - 1; i >= 0; i-- { + elem := &c.stack[i] + if elem.index < elem.count()-1 { + elem.index++ + break + } + } + + // If we've hit the root page then stop and return. This will leave the + // cursor on the last element of the last page. + if i == -1 { + return nil, nil, 0 + } + + // Otherwise start from where we left off in the stack and find the + // first element of the first leaf page. + c.stack = c.stack[:i+1] + c.first() + + // If this is an empty page then restart and move back up the stack. + // https://github.com/boltdb/bolt/issues/450 + if c.stack[len(c.stack)-1].count() == 0 { + continue + } + + return c.keyValue() + } +} + +// search recursively performs a binary search against a given page/node until it finds a given key. +func (c *Cursor) search(key []byte, pgid pgid) { + p, n := c.bucket.pageNode(pgid) + if p != nil && (p.flags&(branchPageFlag|leafPageFlag)) == 0 { + panic(fmt.Sprintf("invalid page type: %d: %x", p.id, p.flags)) + } + e := elemRef{page: p, node: n} + c.stack = append(c.stack, e) + + // If we're on a leaf page/node then find the specific node. + if e.isLeaf() { + c.nsearch(key) + return + } + + if n != nil { + c.searchNode(key, n) + return + } + c.searchPage(key, p) +} + +func (c *Cursor) searchNode(key []byte, n *node) { + var exact bool + index := sort.Search(len(n.inodes), func(i int) bool { + // TODO(benbjohnson): Optimize this range search. It's a bit hacky right now. + // sort.Search() finds the lowest index where f() != -1 but we need the highest index. + ret := bytes.Compare(n.inodes[i].key, key) + if ret == 0 { + exact = true + } + return ret != -1 + }) + if !exact && index > 0 { + index-- + } + c.stack[len(c.stack)-1].index = index + + // Recursively search to the next page. + c.search(key, n.inodes[index].pgid) +} + +func (c *Cursor) searchPage(key []byte, p *page) { + // Binary search for the correct range. + inodes := p.branchPageElements() + + var exact bool + index := sort.Search(int(p.count), func(i int) bool { + // TODO(benbjohnson): Optimize this range search. It's a bit hacky right now. + // sort.Search() finds the lowest index where f() != -1 but we need the highest index. + ret := bytes.Compare(inodes[i].key(), key) + if ret == 0 { + exact = true + } + return ret != -1 + }) + if !exact && index > 0 { + index-- + } + c.stack[len(c.stack)-1].index = index + + // Recursively search to the next page. + c.search(key, inodes[index].pgid) +} + +// nsearch searches the leaf node on the top of the stack for a key. +func (c *Cursor) nsearch(key []byte) { + e := &c.stack[len(c.stack)-1] + p, n := e.page, e.node + + // If we have a node then search its inodes. + if n != nil { + index := sort.Search(len(n.inodes), func(i int) bool { + return bytes.Compare(n.inodes[i].key, key) != -1 + }) + e.index = index + return + } + + // If we have a page then search its leaf elements. + inodes := p.leafPageElements() + index := sort.Search(int(p.count), func(i int) bool { + return bytes.Compare(inodes[i].key(), key) != -1 + }) + e.index = index +} + +// keyValue returns the key and value of the current leaf element. +func (c *Cursor) keyValue() ([]byte, []byte, uint32) { + ref := &c.stack[len(c.stack)-1] + if ref.count() == 0 || ref.index >= ref.count() { + return nil, nil, 0 + } + + // Retrieve value from node. + if ref.node != nil { + inode := &ref.node.inodes[ref.index] + return inode.key, inode.value, inode.flags + } + + // Or retrieve value from page. + elem := ref.page.leafPageElement(uint16(ref.index)) + return elem.key(), elem.value(), elem.flags +} + +// node returns the node that the cursor is currently positioned on. +func (c *Cursor) node() *node { + _assert(len(c.stack) > 0, "accessing a node with a zero-length cursor stack") + + // If the top of the stack is a leaf node then just return it. + if ref := &c.stack[len(c.stack)-1]; ref.node != nil && ref.isLeaf() { + return ref.node + } + + // Start from root and traverse down the hierarchy. + var n = c.stack[0].node + if n == nil { + n = c.bucket.node(c.stack[0].page.id, nil) + } + for _, ref := range c.stack[:len(c.stack)-1] { + _assert(!n.isLeaf, "expected branch node") + n = n.childAt(int(ref.index)) + } + _assert(n.isLeaf, "expected leaf node") + return n +} + +// elemRef represents a reference to an element on a given page/node. +type elemRef struct { + page *page + node *node + index int +} + +// isLeaf returns whether the ref is pointing at a leaf page/node. +func (r *elemRef) isLeaf() bool { + if r.node != nil { + return r.node.isLeaf + } + return (r.page.flags & leafPageFlag) != 0 +} + +// count returns the number of inodes or page elements. +func (r *elemRef) count() int { + if r.node != nil { + return len(r.node.inodes) + } + return int(r.page.count) +} diff --git a/vendor/github.com/coreos/bbolt/cursor_test.go b/vendor/github.com/coreos/bbolt/cursor_test.go new file mode 100644 index 0000000..562d60f --- /dev/null +++ b/vendor/github.com/coreos/bbolt/cursor_test.go @@ -0,0 +1,817 @@ +package bolt_test + +import ( + "bytes" + "encoding/binary" + "fmt" + "log" + "os" + "reflect" + "sort" + "testing" + "testing/quick" + + "github.com/boltdb/bolt" +) + +// Ensure that a cursor can return a reference to the bucket that created it. +func TestCursor_Bucket(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if cb := b.Cursor().Bucket(); !reflect.DeepEqual(cb, b) { + t.Fatal("cursor bucket mismatch") + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a Tx cursor can seek to the appropriate keys. +func TestCursor_Seek(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte("0001")); err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("bar"), []byte("0002")); err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("baz"), []byte("0003")); err != nil { + t.Fatal(err) + } + + if _, err := b.CreateBucket([]byte("bkt")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.View(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte("widgets")).Cursor() + + // Exact match should go to the key. + if k, v := c.Seek([]byte("bar")); !bytes.Equal(k, []byte("bar")) { + t.Fatalf("unexpected key: %v", k) + } else if !bytes.Equal(v, []byte("0002")) { + t.Fatalf("unexpected value: %v", v) + } + + // Inexact match should go to the next key. + if k, v := c.Seek([]byte("bas")); !bytes.Equal(k, []byte("baz")) { + t.Fatalf("unexpected key: %v", k) + } else if !bytes.Equal(v, []byte("0003")) { + t.Fatalf("unexpected value: %v", v) + } + + // Low key should go to the first key. + if k, v := c.Seek([]byte("")); !bytes.Equal(k, []byte("bar")) { + t.Fatalf("unexpected key: %v", k) + } else if !bytes.Equal(v, []byte("0002")) { + t.Fatalf("unexpected value: %v", v) + } + + // High key should return no key. + if k, v := c.Seek([]byte("zzz")); k != nil { + t.Fatalf("expected nil key: %v", k) + } else if v != nil { + t.Fatalf("expected nil value: %v", v) + } + + // Buckets should return their key but no value. + if k, v := c.Seek([]byte("bkt")); !bytes.Equal(k, []byte("bkt")) { + t.Fatalf("unexpected key: %v", k) + } else if v != nil { + t.Fatalf("expected nil value: %v", v) + } + + return nil + }); err != nil { + t.Fatal(err) + } +} + +func TestCursor_Delete(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + const count = 1000 + + // Insert every other key between 0 and $count. + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + for i := 0; i < count; i += 1 { + k := make([]byte, 8) + binary.BigEndian.PutUint64(k, uint64(i)) + if err := b.Put(k, make([]byte, 100)); err != nil { + t.Fatal(err) + } + } + if _, err := b.CreateBucket([]byte("sub")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.Update(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte("widgets")).Cursor() + bound := make([]byte, 8) + binary.BigEndian.PutUint64(bound, uint64(count/2)) + for key, _ := c.First(); bytes.Compare(key, bound) < 0; key, _ = c.Next() { + if err := c.Delete(); err != nil { + t.Fatal(err) + } + } + + c.Seek([]byte("sub")) + if err := c.Delete(); err != bolt.ErrIncompatibleValue { + t.Fatalf("unexpected error: %s", err) + } + + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.View(func(tx *bolt.Tx) error { + stats := tx.Bucket([]byte("widgets")).Stats() + if stats.KeyN != count/2+1 { + t.Fatalf("unexpected KeyN: %d", stats.KeyN) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a Tx cursor can seek to the appropriate keys when there are a +// large number of keys. This test also checks that seek will always move +// forward to the next key. +// +// Related: https://github.com/boltdb/bolt/pull/187 +func TestCursor_Seek_Large(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + var count = 10000 + + // Insert every other key between 0 and $count. + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < count; i += 100 { + for j := i; j < i+100; j += 2 { + k := make([]byte, 8) + binary.BigEndian.PutUint64(k, uint64(j)) + if err := b.Put(k, make([]byte, 100)); err != nil { + t.Fatal(err) + } + } + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.View(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte("widgets")).Cursor() + for i := 0; i < count; i++ { + seek := make([]byte, 8) + binary.BigEndian.PutUint64(seek, uint64(i)) + + k, _ := c.Seek(seek) + + // The last seek is beyond the end of the the range so + // it should return nil. + if i == count-1 { + if k != nil { + t.Fatal("expected nil key") + } + continue + } + + // Otherwise we should seek to the exact key or the next key. + num := binary.BigEndian.Uint64(k) + if i%2 == 0 { + if num != uint64(i) { + t.Fatalf("unexpected num: %d", num) + } + } else { + if num != uint64(i+1) { + t.Fatalf("unexpected num: %d", num) + } + } + } + + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a cursor can iterate over an empty bucket without error. +func TestCursor_EmptyBucket(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucket([]byte("widgets")) + return err + }); err != nil { + t.Fatal(err) + } + + if err := db.View(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte("widgets")).Cursor() + k, v := c.First() + if k != nil { + t.Fatalf("unexpected key: %v", k) + } else if v != nil { + t.Fatalf("unexpected value: %v", v) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a Tx cursor can reverse iterate over an empty bucket without error. +func TestCursor_EmptyBucketReverse(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucket([]byte("widgets")) + return err + }); err != nil { + t.Fatal(err) + } + if err := db.View(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte("widgets")).Cursor() + k, v := c.Last() + if k != nil { + t.Fatalf("unexpected key: %v", k) + } else if v != nil { + t.Fatalf("unexpected value: %v", v) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a Tx cursor can iterate over a single root with a couple elements. +func TestCursor_Iterate_Leaf(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("baz"), []byte{}); err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte{0}); err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("bar"), []byte{1}); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + tx, err := db.Begin(false) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback() }() + + c := tx.Bucket([]byte("widgets")).Cursor() + + k, v := c.First() + if !bytes.Equal(k, []byte("bar")) { + t.Fatalf("unexpected key: %v", k) + } else if !bytes.Equal(v, []byte{1}) { + t.Fatalf("unexpected value: %v", v) + } + + k, v = c.Next() + if !bytes.Equal(k, []byte("baz")) { + t.Fatalf("unexpected key: %v", k) + } else if !bytes.Equal(v, []byte{}) { + t.Fatalf("unexpected value: %v", v) + } + + k, v = c.Next() + if !bytes.Equal(k, []byte("foo")) { + t.Fatalf("unexpected key: %v", k) + } else if !bytes.Equal(v, []byte{0}) { + t.Fatalf("unexpected value: %v", v) + } + + k, v = c.Next() + if k != nil { + t.Fatalf("expected nil key: %v", k) + } else if v != nil { + t.Fatalf("expected nil value: %v", v) + } + + k, v = c.Next() + if k != nil { + t.Fatalf("expected nil key: %v", k) + } else if v != nil { + t.Fatalf("expected nil value: %v", v) + } + + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } +} + +// Ensure that a Tx cursor can iterate in reverse over a single root with a couple elements. +func TestCursor_LeafRootReverse(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("baz"), []byte{}); err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte{0}); err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("bar"), []byte{1}); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + tx, err := db.Begin(false) + if err != nil { + t.Fatal(err) + } + c := tx.Bucket([]byte("widgets")).Cursor() + + if k, v := c.Last(); !bytes.Equal(k, []byte("foo")) { + t.Fatalf("unexpected key: %v", k) + } else if !bytes.Equal(v, []byte{0}) { + t.Fatalf("unexpected value: %v", v) + } + + if k, v := c.Prev(); !bytes.Equal(k, []byte("baz")) { + t.Fatalf("unexpected key: %v", k) + } else if !bytes.Equal(v, []byte{}) { + t.Fatalf("unexpected value: %v", v) + } + + if k, v := c.Prev(); !bytes.Equal(k, []byte("bar")) { + t.Fatalf("unexpected key: %v", k) + } else if !bytes.Equal(v, []byte{1}) { + t.Fatalf("unexpected value: %v", v) + } + + if k, v := c.Prev(); k != nil { + t.Fatalf("expected nil key: %v", k) + } else if v != nil { + t.Fatalf("expected nil value: %v", v) + } + + if k, v := c.Prev(); k != nil { + t.Fatalf("expected nil key: %v", k) + } else if v != nil { + t.Fatalf("expected nil value: %v", v) + } + + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } +} + +// Ensure that a Tx cursor can restart from the beginning. +func TestCursor_Restart(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("bar"), []byte{}); err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte{}); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + tx, err := db.Begin(false) + if err != nil { + t.Fatal(err) + } + c := tx.Bucket([]byte("widgets")).Cursor() + + if k, _ := c.First(); !bytes.Equal(k, []byte("bar")) { + t.Fatalf("unexpected key: %v", k) + } + if k, _ := c.Next(); !bytes.Equal(k, []byte("foo")) { + t.Fatalf("unexpected key: %v", k) + } + + if k, _ := c.First(); !bytes.Equal(k, []byte("bar")) { + t.Fatalf("unexpected key: %v", k) + } + if k, _ := c.Next(); !bytes.Equal(k, []byte("foo")) { + t.Fatalf("unexpected key: %v", k) + } + + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } +} + +// Ensure that a cursor can skip over empty pages that have been deleted. +func TestCursor_First_EmptyPages(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + // Create 1000 keys in the "widgets" bucket. + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < 1000; i++ { + if err := b.Put(u64tob(uint64(i)), []byte{}); err != nil { + t.Fatal(err) + } + } + + return nil + }); err != nil { + t.Fatal(err) + } + + // Delete half the keys and then try to iterate. + if err := db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("widgets")) + for i := 0; i < 600; i++ { + if err := b.Delete(u64tob(uint64(i))); err != nil { + t.Fatal(err) + } + } + + c := b.Cursor() + var n int + for k, _ := c.First(); k != nil; k, _ = c.Next() { + n++ + } + if n != 400 { + t.Fatalf("unexpected key count: %d", n) + } + + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a Tx can iterate over all elements in a bucket. +func TestCursor_QuickCheck(t *testing.T) { + f := func(items testdata) bool { + db := MustOpenDB() + defer db.MustClose() + + // Bulk insert all values. + tx, err := db.Begin(true) + if err != nil { + t.Fatal(err) + } + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + for _, item := range items { + if err := b.Put(item.Key, item.Value); err != nil { + t.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + + // Sort test data. + sort.Sort(items) + + // Iterate over all items and check consistency. + var index = 0 + tx, err = db.Begin(false) + if err != nil { + t.Fatal(err) + } + + c := tx.Bucket([]byte("widgets")).Cursor() + for k, v := c.First(); k != nil && index < len(items); k, v = c.Next() { + if !bytes.Equal(k, items[index].Key) { + t.Fatalf("unexpected key: %v", k) + } else if !bytes.Equal(v, items[index].Value) { + t.Fatalf("unexpected value: %v", v) + } + index++ + } + if len(items) != index { + t.Fatalf("unexpected item count: %v, expected %v", len(items), index) + } + + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + + return true + } + if err := quick.Check(f, qconfig()); err != nil { + t.Error(err) + } +} + +// Ensure that a transaction can iterate over all elements in a bucket in reverse. +func TestCursor_QuickCheck_Reverse(t *testing.T) { + f := func(items testdata) bool { + db := MustOpenDB() + defer db.MustClose() + + // Bulk insert all values. + tx, err := db.Begin(true) + if err != nil { + t.Fatal(err) + } + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + for _, item := range items { + if err := b.Put(item.Key, item.Value); err != nil { + t.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + + // Sort test data. + sort.Sort(revtestdata(items)) + + // Iterate over all items and check consistency. + var index = 0 + tx, err = db.Begin(false) + if err != nil { + t.Fatal(err) + } + c := tx.Bucket([]byte("widgets")).Cursor() + for k, v := c.Last(); k != nil && index < len(items); k, v = c.Prev() { + if !bytes.Equal(k, items[index].Key) { + t.Fatalf("unexpected key: %v", k) + } else if !bytes.Equal(v, items[index].Value) { + t.Fatalf("unexpected value: %v", v) + } + index++ + } + if len(items) != index { + t.Fatalf("unexpected item count: %v, expected %v", len(items), index) + } + + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + + return true + } + if err := quick.Check(f, qconfig()); err != nil { + t.Error(err) + } +} + +// Ensure that a Tx cursor can iterate over subbuckets. +func TestCursor_QuickCheck_BucketsOnly(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if _, err := b.CreateBucket([]byte("foo")); err != nil { + t.Fatal(err) + } + if _, err := b.CreateBucket([]byte("bar")); err != nil { + t.Fatal(err) + } + if _, err := b.CreateBucket([]byte("baz")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.View(func(tx *bolt.Tx) error { + var names []string + c := tx.Bucket([]byte("widgets")).Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + names = append(names, string(k)) + if v != nil { + t.Fatalf("unexpected value: %v", v) + } + } + if !reflect.DeepEqual(names, []string{"bar", "baz", "foo"}) { + t.Fatalf("unexpected names: %+v", names) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a Tx cursor can reverse iterate over subbuckets. +func TestCursor_QuickCheck_BucketsOnly_Reverse(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if _, err := b.CreateBucket([]byte("foo")); err != nil { + t.Fatal(err) + } + if _, err := b.CreateBucket([]byte("bar")); err != nil { + t.Fatal(err) + } + if _, err := b.CreateBucket([]byte("baz")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.View(func(tx *bolt.Tx) error { + var names []string + c := tx.Bucket([]byte("widgets")).Cursor() + for k, v := c.Last(); k != nil; k, v = c.Prev() { + names = append(names, string(k)) + if v != nil { + t.Fatalf("unexpected value: %v", v) + } + } + if !reflect.DeepEqual(names, []string{"foo", "baz", "bar"}) { + t.Fatalf("unexpected names: %+v", names) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +func ExampleCursor() { + // Open the database. + db, err := bolt.Open(tempfile(), 0666, nil) + if err != nil { + log.Fatal(err) + } + defer os.Remove(db.Path()) + + // Start a read-write transaction. + if err := db.Update(func(tx *bolt.Tx) error { + // Create a new bucket. + b, err := tx.CreateBucket([]byte("animals")) + if err != nil { + return err + } + + // Insert data into a bucket. + if err := b.Put([]byte("dog"), []byte("fun")); err != nil { + log.Fatal(err) + } + if err := b.Put([]byte("cat"), []byte("lame")); err != nil { + log.Fatal(err) + } + if err := b.Put([]byte("liger"), []byte("awesome")); err != nil { + log.Fatal(err) + } + + // Create a cursor for iteration. + c := b.Cursor() + + // Iterate over items in sorted key order. This starts from the + // first key/value pair and updates the k/v variables to the + // next key/value on each iteration. + // + // The loop finishes at the end of the cursor when a nil key is returned. + for k, v := c.First(); k != nil; k, v = c.Next() { + fmt.Printf("A %s is %s.\n", k, v) + } + + return nil + }); err != nil { + log.Fatal(err) + } + + if err := db.Close(); err != nil { + log.Fatal(err) + } + + // Output: + // A cat is lame. + // A dog is fun. + // A liger is awesome. +} + +func ExampleCursor_reverse() { + // Open the database. + db, err := bolt.Open(tempfile(), 0666, nil) + if err != nil { + log.Fatal(err) + } + defer os.Remove(db.Path()) + + // Start a read-write transaction. + if err := db.Update(func(tx *bolt.Tx) error { + // Create a new bucket. + b, err := tx.CreateBucket([]byte("animals")) + if err != nil { + return err + } + + // Insert data into a bucket. + if err := b.Put([]byte("dog"), []byte("fun")); err != nil { + log.Fatal(err) + } + if err := b.Put([]byte("cat"), []byte("lame")); err != nil { + log.Fatal(err) + } + if err := b.Put([]byte("liger"), []byte("awesome")); err != nil { + log.Fatal(err) + } + + // Create a cursor for iteration. + c := b.Cursor() + + // Iterate over items in reverse sorted key order. This starts + // from the last key/value pair and updates the k/v variables to + // the previous key/value on each iteration. + // + // The loop finishes at the beginning of the cursor when a nil key + // is returned. + for k, v := c.Last(); k != nil; k, v = c.Prev() { + fmt.Printf("A %s is %s.\n", k, v) + } + + return nil + }); err != nil { + log.Fatal(err) + } + + // Close the database to release the file lock. + if err := db.Close(); err != nil { + log.Fatal(err) + } + + // Output: + // A liger is awesome. + // A dog is fun. + // A cat is lame. +} diff --git a/vendor/github.com/coreos/bbolt/db.go b/vendor/github.com/coreos/bbolt/db.go new file mode 100644 index 0000000..1223493 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/db.go @@ -0,0 +1,1036 @@ +package bolt + +import ( + "errors" + "fmt" + "hash/fnv" + "log" + "os" + "runtime" + "runtime/debug" + "strings" + "sync" + "time" + "unsafe" +) + +// The largest step that can be taken when remapping the mmap. +const maxMmapStep = 1 << 30 // 1GB + +// The data file format version. +const version = 2 + +// Represents a marker value to indicate that a file is a Bolt DB. +const magic uint32 = 0xED0CDAED + +// IgnoreNoSync specifies whether the NoSync field of a DB is ignored when +// syncing changes to a file. This is required as some operating systems, +// such as OpenBSD, do not have a unified buffer cache (UBC) and writes +// must be synchronized using the msync(2) syscall. +const IgnoreNoSync = runtime.GOOS == "openbsd" + +// Default values if not set in a DB instance. +const ( + DefaultMaxBatchSize int = 1000 + DefaultMaxBatchDelay = 10 * time.Millisecond + DefaultAllocSize = 16 * 1024 * 1024 +) + +// default page size for db is set to the OS page size. +var defaultPageSize = os.Getpagesize() + +// DB represents a collection of buckets persisted to a file on disk. +// All data access is performed through transactions which can be obtained through the DB. +// All the functions on DB will return a ErrDatabaseNotOpen if accessed before Open() is called. +type DB struct { + // When enabled, the database will perform a Check() after every commit. + // A panic is issued if the database is in an inconsistent state. This + // flag has a large performance impact so it should only be used for + // debugging purposes. + StrictMode bool + + // Setting the NoSync flag will cause the database to skip fsync() + // calls after each commit. This can be useful when bulk loading data + // into a database and you can restart the bulk load in the event of + // a system failure or database corruption. Do not set this flag for + // normal use. + // + // If the package global IgnoreNoSync constant is true, this value is + // ignored. See the comment on that constant for more details. + // + // THIS IS UNSAFE. PLEASE USE WITH CAUTION. + NoSync bool + + // When true, skips the truncate call when growing the database. + // Setting this to true is only safe on non-ext3/ext4 systems. + // Skipping truncation avoids preallocation of hard drive space and + // bypasses a truncate() and fsync() syscall on remapping. + // + // https://github.com/boltdb/bolt/issues/284 + NoGrowSync bool + + // If you want to read the entire database fast, you can set MmapFlag to + // syscall.MAP_POPULATE on Linux 2.6.23+ for sequential read-ahead. + MmapFlags int + + // MaxBatchSize is the maximum size of a batch. Default value is + // copied from DefaultMaxBatchSize in Open. + // + // If <=0, disables batching. + // + // Do not change concurrently with calls to Batch. + MaxBatchSize int + + // MaxBatchDelay is the maximum delay before a batch starts. + // Default value is copied from DefaultMaxBatchDelay in Open. + // + // If <=0, effectively disables batching. + // + // Do not change concurrently with calls to Batch. + MaxBatchDelay time.Duration + + // AllocSize is the amount of space allocated when the database + // needs to create new pages. This is done to amortize the cost + // of truncate() and fsync() when growing the data file. + AllocSize int + + path string + file *os.File + lockfile *os.File // windows only + dataref []byte // mmap'ed readonly, write throws SEGV + data *[maxMapSize]byte + datasz int + filesz int // current on disk file size + meta0 *meta + meta1 *meta + pageSize int + opened bool + rwtx *Tx + txs []*Tx + freelist *freelist + stats Stats + + pagePool sync.Pool + + batchMu sync.Mutex + batch *batch + + rwlock sync.Mutex // Allows only one writer at a time. + metalock sync.Mutex // Protects meta page access. + mmaplock sync.RWMutex // Protects mmap access during remapping. + statlock sync.RWMutex // Protects stats access. + + ops struct { + writeAt func(b []byte, off int64) (n int, err error) + } + + // Read only mode. + // When true, Update() and Begin(true) return ErrDatabaseReadOnly immediately. + readOnly bool +} + +// Path returns the path to currently open database file. +func (db *DB) Path() string { + return db.path +} + +// GoString returns the Go string representation of the database. +func (db *DB) GoString() string { + return fmt.Sprintf("bolt.DB{path:%q}", db.path) +} + +// String returns the string representation of the database. +func (db *DB) String() string { + return fmt.Sprintf("DB<%q>", db.path) +} + +// Open creates and opens a database at the given path. +// If the file does not exist then it will be created automatically. +// Passing in nil options will cause Bolt to open the database with the default options. +func Open(path string, mode os.FileMode, options *Options) (*DB, error) { + var db = &DB{opened: true} + + // Set default options if no options are provided. + if options == nil { + options = DefaultOptions + } + db.NoGrowSync = options.NoGrowSync + db.MmapFlags = options.MmapFlags + + // Set default values for later DB operations. + db.MaxBatchSize = DefaultMaxBatchSize + db.MaxBatchDelay = DefaultMaxBatchDelay + db.AllocSize = DefaultAllocSize + + flag := os.O_RDWR + if options.ReadOnly { + flag = os.O_RDONLY + db.readOnly = true + } + + // Open data file and separate sync handler for metadata writes. + db.path = path + var err error + if db.file, err = os.OpenFile(db.path, flag|os.O_CREATE, mode); err != nil { + _ = db.close() + return nil, err + } + + // Lock file so that other processes using Bolt in read-write mode cannot + // use the database at the same time. This would cause corruption since + // the two processes would write meta pages and free pages separately. + // The database file is locked exclusively (only one process can grab the lock) + // if !options.ReadOnly. + // The database file is locked using the shared lock (more than one process may + // hold a lock at the same time) otherwise (options.ReadOnly is set). + if err := flock(db, mode, !db.readOnly, options.Timeout); err != nil { + _ = db.close() + return nil, err + } + + // Default values for test hooks + db.ops.writeAt = db.file.WriteAt + + // Initialize the database if it doesn't exist. + if info, err := db.file.Stat(); err != nil { + return nil, err + } else if info.Size() == 0 { + // Initialize new files with meta pages. + if err := db.init(); err != nil { + return nil, err + } + } else { + // Read the first meta page to determine the page size. + var buf [0x1000]byte + if _, err := db.file.ReadAt(buf[:], 0); err == nil { + m := db.pageInBuffer(buf[:], 0).meta() + if err := m.validate(); err != nil { + // If we can't read the page size, we can assume it's the same + // as the OS -- since that's how the page size was chosen in the + // first place. + // + // If the first page is invalid and this OS uses a different + // page size than what the database was created with then we + // are out of luck and cannot access the database. + db.pageSize = os.Getpagesize() + } else { + db.pageSize = int(m.pageSize) + } + } + } + + // Initialize page pool. + db.pagePool = sync.Pool{ + New: func() interface{} { + return make([]byte, db.pageSize) + }, + } + + // Memory map the data file. + if err := db.mmap(options.InitialMmapSize); err != nil { + _ = db.close() + return nil, err + } + + // Read in the freelist. + db.freelist = newFreelist() + db.freelist.read(db.page(db.meta().freelist)) + + // Mark the database as opened and return. + return db, nil +} + +// mmap opens the underlying memory-mapped file and initializes the meta references. +// minsz is the minimum size that the new mmap can be. +func (db *DB) mmap(minsz int) error { + db.mmaplock.Lock() + defer db.mmaplock.Unlock() + + info, err := db.file.Stat() + if err != nil { + return fmt.Errorf("mmap stat error: %s", err) + } else if int(info.Size()) < db.pageSize*2 { + return fmt.Errorf("file size too small") + } + + // Ensure the size is at least the minimum size. + var size = int(info.Size()) + if size < minsz { + size = minsz + } + size, err = db.mmapSize(size) + if err != nil { + return err + } + + // Dereference all mmap references before unmapping. + if db.rwtx != nil { + db.rwtx.root.dereference() + } + + // Unmap existing data before continuing. + if err := db.munmap(); err != nil { + return err + } + + // Memory-map the data file as a byte slice. + if err := mmap(db, size); err != nil { + return err + } + + // Save references to the meta pages. + db.meta0 = db.page(0).meta() + db.meta1 = db.page(1).meta() + + // Validate the meta pages. We only return an error if both meta pages fail + // validation, since meta0 failing validation means that it wasn't saved + // properly -- but we can recover using meta1. And vice-versa. + err0 := db.meta0.validate() + err1 := db.meta1.validate() + if err0 != nil && err1 != nil { + return err0 + } + + return nil +} + +// munmap unmaps the data file from memory. +func (db *DB) munmap() error { + if err := munmap(db); err != nil { + return fmt.Errorf("unmap error: " + err.Error()) + } + return nil +} + +// mmapSize determines the appropriate size for the mmap given the current size +// of the database. The minimum size is 32KB and doubles until it reaches 1GB. +// Returns an error if the new mmap size is greater than the max allowed. +func (db *DB) mmapSize(size int) (int, error) { + // Double the size from 32KB until 1GB. + for i := uint(15); i <= 30; i++ { + if size <= 1< maxMapSize { + return 0, fmt.Errorf("mmap too large") + } + + // If larger than 1GB then grow by 1GB at a time. + sz := int64(size) + if remainder := sz % int64(maxMmapStep); remainder > 0 { + sz += int64(maxMmapStep) - remainder + } + + // Ensure that the mmap size is a multiple of the page size. + // This should always be true since we're incrementing in MBs. + pageSize := int64(db.pageSize) + if (sz % pageSize) != 0 { + sz = ((sz / pageSize) + 1) * pageSize + } + + // If we've exceeded the max size then only grow up to the max size. + if sz > maxMapSize { + sz = maxMapSize + } + + return int(sz), nil +} + +// init creates a new database file and initializes its meta pages. +func (db *DB) init() error { + // Set the page size to the OS page size. + db.pageSize = os.Getpagesize() + + // Create two meta pages on a buffer. + buf := make([]byte, db.pageSize*4) + for i := 0; i < 2; i++ { + p := db.pageInBuffer(buf[:], pgid(i)) + p.id = pgid(i) + p.flags = metaPageFlag + + // Initialize the meta page. + m := p.meta() + m.magic = magic + m.version = version + m.pageSize = uint32(db.pageSize) + m.freelist = 2 + m.root = bucket{root: 3} + m.pgid = 4 + m.txid = txid(i) + m.checksum = m.sum64() + } + + // Write an empty freelist at page 3. + p := db.pageInBuffer(buf[:], pgid(2)) + p.id = pgid(2) + p.flags = freelistPageFlag + p.count = 0 + + // Write an empty leaf page at page 4. + p = db.pageInBuffer(buf[:], pgid(3)) + p.id = pgid(3) + p.flags = leafPageFlag + p.count = 0 + + // Write the buffer to our data file. + if _, err := db.ops.writeAt(buf, 0); err != nil { + return err + } + if err := fdatasync(db); err != nil { + return err + } + + return nil +} + +// Close releases all database resources. +// All transactions must be closed before closing the database. +func (db *DB) Close() error { + db.rwlock.Lock() + defer db.rwlock.Unlock() + + db.metalock.Lock() + defer db.metalock.Unlock() + + db.mmaplock.RLock() + defer db.mmaplock.RUnlock() + + return db.close() +} + +func (db *DB) close() error { + if !db.opened { + return nil + } + + db.opened = false + + db.freelist = nil + + // Clear ops. + db.ops.writeAt = nil + + // Close the mmap. + if err := db.munmap(); err != nil { + return err + } + + // Close file handles. + if db.file != nil { + // No need to unlock read-only file. + if !db.readOnly { + // Unlock the file. + if err := funlock(db); err != nil { + log.Printf("bolt.Close(): funlock error: %s", err) + } + } + + // Close the file descriptor. + if err := db.file.Close(); err != nil { + return fmt.Errorf("db file close: %s", err) + } + db.file = nil + } + + db.path = "" + return nil +} + +// Begin starts a new transaction. +// Multiple read-only transactions can be used concurrently but only one +// write transaction can be used at a time. Starting multiple write transactions +// will cause the calls to block and be serialized until the current write +// transaction finishes. +// +// Transactions should not be dependent on one another. Opening a read +// transaction and a write transaction in the same goroutine can cause the +// writer to deadlock because the database periodically needs to re-mmap itself +// as it grows and it cannot do that while a read transaction is open. +// +// If a long running read transaction (for example, a snapshot transaction) is +// needed, you might want to set DB.InitialMmapSize to a large enough value +// to avoid potential blocking of write transaction. +// +// IMPORTANT: You must close read-only transactions after you are finished or +// else the database will not reclaim old pages. +func (db *DB) Begin(writable bool) (*Tx, error) { + if writable { + return db.beginRWTx() + } + return db.beginTx() +} + +func (db *DB) beginTx() (*Tx, error) { + // Lock the meta pages while we initialize the transaction. We obtain + // the meta lock before the mmap lock because that's the order that the + // write transaction will obtain them. + db.metalock.Lock() + + // Obtain a read-only lock on the mmap. When the mmap is remapped it will + // obtain a write lock so all transactions must finish before it can be + // remapped. + db.mmaplock.RLock() + + // Exit if the database is not open yet. + if !db.opened { + db.mmaplock.RUnlock() + db.metalock.Unlock() + return nil, ErrDatabaseNotOpen + } + + // Create a transaction associated with the database. + t := &Tx{} + t.init(db) + + // Keep track of transaction until it closes. + db.txs = append(db.txs, t) + n := len(db.txs) + + // Unlock the meta pages. + db.metalock.Unlock() + + // Update the transaction stats. + db.statlock.Lock() + db.stats.TxN++ + db.stats.OpenTxN = n + db.statlock.Unlock() + + return t, nil +} + +func (db *DB) beginRWTx() (*Tx, error) { + // If the database was opened with Options.ReadOnly, return an error. + if db.readOnly { + return nil, ErrDatabaseReadOnly + } + + // Obtain writer lock. This is released by the transaction when it closes. + // This enforces only one writer transaction at a time. + db.rwlock.Lock() + + // Once we have the writer lock then we can lock the meta pages so that + // we can set up the transaction. + db.metalock.Lock() + defer db.metalock.Unlock() + + // Exit if the database is not open yet. + if !db.opened { + db.rwlock.Unlock() + return nil, ErrDatabaseNotOpen + } + + // Create a transaction associated with the database. + t := &Tx{writable: true} + t.init(db) + db.rwtx = t + + // Free any pages associated with closed read-only transactions. + var minid txid = 0xFFFFFFFFFFFFFFFF + for _, t := range db.txs { + if t.meta.txid < minid { + minid = t.meta.txid + } + } + if minid > 0 { + db.freelist.release(minid - 1) + } + + return t, nil +} + +// removeTx removes a transaction from the database. +func (db *DB) removeTx(tx *Tx) { + // Release the read lock on the mmap. + db.mmaplock.RUnlock() + + // Use the meta lock to restrict access to the DB object. + db.metalock.Lock() + + // Remove the transaction. + for i, t := range db.txs { + if t == tx { + db.txs = append(db.txs[:i], db.txs[i+1:]...) + break + } + } + n := len(db.txs) + + // Unlock the meta pages. + db.metalock.Unlock() + + // Merge statistics. + db.statlock.Lock() + db.stats.OpenTxN = n + db.stats.TxStats.add(&tx.stats) + db.statlock.Unlock() +} + +// Update executes a function within the context of a read-write managed transaction. +// If no error is returned from the function then the transaction is committed. +// If an error is returned then the entire transaction is rolled back. +// Any error that is returned from the function or returned from the commit is +// returned from the Update() method. +// +// Attempting to manually commit or rollback within the function will cause a panic. +func (db *DB) Update(fn func(*Tx) error) error { + t, err := db.Begin(true) + if err != nil { + return err + } + + // Make sure the transaction rolls back in the event of a panic. + defer func() { + if t.db != nil { + t.rollback() + } + }() + + // Mark as a managed tx so that the inner function cannot manually commit. + t.managed = true + + // If an error is returned from the function then rollback and return error. + err = fn(t) + t.managed = false + if err != nil { + _ = t.Rollback() + return err + } + + return t.Commit() +} + +// View executes a function within the context of a managed read-only transaction. +// Any error that is returned from the function is returned from the View() method. +// +// Attempting to manually rollback within the function will cause a panic. +func (db *DB) View(fn func(*Tx) error) error { + t, err := db.Begin(false) + if err != nil { + return err + } + + // Make sure the transaction rolls back in the event of a panic. + defer func() { + if t.db != nil { + t.rollback() + } + }() + + // Mark as a managed tx so that the inner function cannot manually rollback. + t.managed = true + + // If an error is returned from the function then pass it through. + err = fn(t) + t.managed = false + if err != nil { + _ = t.Rollback() + return err + } + + if err := t.Rollback(); err != nil { + return err + } + + return nil +} + +// Batch calls fn as part of a batch. It behaves similar to Update, +// except: +// +// 1. concurrent Batch calls can be combined into a single Bolt +// transaction. +// +// 2. the function passed to Batch may be called multiple times, +// regardless of whether it returns error or not. +// +// This means that Batch function side effects must be idempotent and +// take permanent effect only after a successful return is seen in +// caller. +// +// The maximum batch size and delay can be adjusted with DB.MaxBatchSize +// and DB.MaxBatchDelay, respectively. +// +// Batch is only useful when there are multiple goroutines calling it. +func (db *DB) Batch(fn func(*Tx) error) error { + errCh := make(chan error, 1) + + db.batchMu.Lock() + if (db.batch == nil) || (db.batch != nil && len(db.batch.calls) >= db.MaxBatchSize) { + // There is no existing batch, or the existing batch is full; start a new one. + db.batch = &batch{ + db: db, + } + db.batch.timer = time.AfterFunc(db.MaxBatchDelay, db.batch.trigger) + } + db.batch.calls = append(db.batch.calls, call{fn: fn, err: errCh}) + if len(db.batch.calls) >= db.MaxBatchSize { + // wake up batch, it's ready to run + go db.batch.trigger() + } + db.batchMu.Unlock() + + err := <-errCh + if err == trySolo { + err = db.Update(fn) + } + return err +} + +type call struct { + fn func(*Tx) error + err chan<- error +} + +type batch struct { + db *DB + timer *time.Timer + start sync.Once + calls []call +} + +// trigger runs the batch if it hasn't already been run. +func (b *batch) trigger() { + b.start.Do(b.run) +} + +// run performs the transactions in the batch and communicates results +// back to DB.Batch. +func (b *batch) run() { + b.db.batchMu.Lock() + b.timer.Stop() + // Make sure no new work is added to this batch, but don't break + // other batches. + if b.db.batch == b { + b.db.batch = nil + } + b.db.batchMu.Unlock() + +retry: + for len(b.calls) > 0 { + var failIdx = -1 + err := b.db.Update(func(tx *Tx) error { + for i, c := range b.calls { + if err := safelyCall(c.fn, tx); err != nil { + failIdx = i + return err + } + } + return nil + }) + + if failIdx >= 0 { + // take the failing transaction out of the batch. it's + // safe to shorten b.calls here because db.batch no longer + // points to us, and we hold the mutex anyway. + c := b.calls[failIdx] + b.calls[failIdx], b.calls = b.calls[len(b.calls)-1], b.calls[:len(b.calls)-1] + // tell the submitter re-run it solo, continue with the rest of the batch + c.err <- trySolo + continue retry + } + + // pass success, or bolt internal errors, to all callers + for _, c := range b.calls { + if c.err != nil { + c.err <- err + } + } + break retry + } +} + +// trySolo is a special sentinel error value used for signaling that a +// transaction function should be re-run. It should never be seen by +// callers. +var trySolo = errors.New("batch function returned an error and should be re-run solo") + +type panicked struct { + reason interface{} +} + +func (p panicked) Error() string { + if err, ok := p.reason.(error); ok { + return err.Error() + } + return fmt.Sprintf("panic: %v", p.reason) +} + +func safelyCall(fn func(*Tx) error, tx *Tx) (err error) { + defer func() { + if p := recover(); p != nil { + err = panicked{p} + } + }() + return fn(tx) +} + +// Sync executes fdatasync() against the database file handle. +// +// This is not necessary under normal operation, however, if you use NoSync +// then it allows you to force the database file to sync against the disk. +func (db *DB) Sync() error { return fdatasync(db) } + +// Stats retrieves ongoing performance stats for the database. +// This is only updated when a transaction closes. +func (db *DB) Stats() Stats { + db.statlock.RLock() + defer db.statlock.RUnlock() + return db.stats +} + +// This is for internal access to the raw data bytes from the C cursor, use +// carefully, or not at all. +func (db *DB) Info() *Info { + return &Info{uintptr(unsafe.Pointer(&db.data[0])), db.pageSize} +} + +// page retrieves a page reference from the mmap based on the current page size. +func (db *DB) page(id pgid) *page { + pos := id * pgid(db.pageSize) + return (*page)(unsafe.Pointer(&db.data[pos])) +} + +// pageInBuffer retrieves a page reference from a given byte array based on the current page size. +func (db *DB) pageInBuffer(b []byte, id pgid) *page { + return (*page)(unsafe.Pointer(&b[id*pgid(db.pageSize)])) +} + +// meta retrieves the current meta page reference. +func (db *DB) meta() *meta { + // We have to return the meta with the highest txid which doesn't fail + // validation. Otherwise, we can cause errors when in fact the database is + // in a consistent state. metaA is the one with the higher txid. + metaA := db.meta0 + metaB := db.meta1 + if db.meta1.txid > db.meta0.txid { + metaA = db.meta1 + metaB = db.meta0 + } + + // Use higher meta page if valid. Otherwise fallback to previous, if valid. + if err := metaA.validate(); err == nil { + return metaA + } else if err := metaB.validate(); err == nil { + return metaB + } + + // This should never be reached, because both meta1 and meta0 were validated + // on mmap() and we do fsync() on every write. + panic("bolt.DB.meta(): invalid meta pages") +} + +// allocate returns a contiguous block of memory starting at a given page. +func (db *DB) allocate(count int) (*page, error) { + // Allocate a temporary buffer for the page. + var buf []byte + if count == 1 { + buf = db.pagePool.Get().([]byte) + } else { + buf = make([]byte, count*db.pageSize) + } + p := (*page)(unsafe.Pointer(&buf[0])) + p.overflow = uint32(count - 1) + + // Use pages from the freelist if they are available. + if p.id = db.freelist.allocate(count); p.id != 0 { + return p, nil + } + + // Resize mmap() if we're at the end. + p.id = db.rwtx.meta.pgid + var minsz = int((p.id+pgid(count))+1) * db.pageSize + if minsz >= db.datasz { + if err := db.mmap(minsz); err != nil { + return nil, fmt.Errorf("mmap allocate error: %s", err) + } + } + + // Move the page id high water mark. + db.rwtx.meta.pgid += pgid(count) + + return p, nil +} + +// grow grows the size of the database to the given sz. +func (db *DB) grow(sz int) error { + // Ignore if the new size is less than available file size. + if sz <= db.filesz { + return nil + } + + // If the data is smaller than the alloc size then only allocate what's needed. + // Once it goes over the allocation size then allocate in chunks. + if db.datasz < db.AllocSize { + sz = db.datasz + } else { + sz += db.AllocSize + } + + // Truncate and fsync to ensure file size metadata is flushed. + // https://github.com/boltdb/bolt/issues/284 + if !db.NoGrowSync && !db.readOnly { + if runtime.GOOS != "windows" { + if err := db.file.Truncate(int64(sz)); err != nil { + return fmt.Errorf("file resize error: %s", err) + } + } + if err := db.file.Sync(); err != nil { + return fmt.Errorf("file sync error: %s", err) + } + } + + db.filesz = sz + return nil +} + +func (db *DB) IsReadOnly() bool { + return db.readOnly +} + +// Options represents the options that can be set when opening a database. +type Options struct { + // Timeout is the amount of time to wait to obtain a file lock. + // When set to zero it will wait indefinitely. This option is only + // available on Darwin and Linux. + Timeout time.Duration + + // Sets the DB.NoGrowSync flag before memory mapping the file. + NoGrowSync bool + + // Open database in read-only mode. Uses flock(..., LOCK_SH |LOCK_NB) to + // grab a shared lock (UNIX). + ReadOnly bool + + // Sets the DB.MmapFlags flag before memory mapping the file. + MmapFlags int + + // InitialMmapSize is the initial mmap size of the database + // in bytes. Read transactions won't block write transaction + // if the InitialMmapSize is large enough to hold database mmap + // size. (See DB.Begin for more information) + // + // If <=0, the initial map size is 0. + // If initialMmapSize is smaller than the previous database size, + // it takes no effect. + InitialMmapSize int +} + +// DefaultOptions represent the options used if nil options are passed into Open(). +// No timeout is used which will cause Bolt to wait indefinitely for a lock. +var DefaultOptions = &Options{ + Timeout: 0, + NoGrowSync: false, +} + +// Stats represents statistics about the database. +type Stats struct { + // Freelist stats + FreePageN int // total number of free pages on the freelist + PendingPageN int // total number of pending pages on the freelist + FreeAlloc int // total bytes allocated in free pages + FreelistInuse int // total bytes used by the freelist + + // Transaction stats + TxN int // total number of started read transactions + OpenTxN int // number of currently open read transactions + + TxStats TxStats // global, ongoing stats. +} + +// Sub calculates and returns the difference between two sets of database stats. +// This is useful when obtaining stats at two different points and time and +// you need the performance counters that occurred within that time span. +func (s *Stats) Sub(other *Stats) Stats { + if other == nil { + return *s + } + var diff Stats + diff.FreePageN = s.FreePageN + diff.PendingPageN = s.PendingPageN + diff.FreeAlloc = s.FreeAlloc + diff.FreelistInuse = s.FreelistInuse + diff.TxN = other.TxN - s.TxN + diff.TxStats = s.TxStats.Sub(&other.TxStats) + return diff +} + +func (s *Stats) add(other *Stats) { + s.TxStats.add(&other.TxStats) +} + +type Info struct { + Data uintptr + PageSize int +} + +type meta struct { + magic uint32 + version uint32 + pageSize uint32 + flags uint32 + root bucket + freelist pgid + pgid pgid + txid txid + checksum uint64 +} + +// validate checks the marker bytes and version of the meta page to ensure it matches this binary. +func (m *meta) validate() error { + if m.magic != magic { + return ErrInvalid + } else if m.version != version { + return ErrVersionMismatch + } else if m.checksum != 0 && m.checksum != m.sum64() { + return ErrChecksum + } + return nil +} + +// copy copies one meta object to another. +func (m *meta) copy(dest *meta) { + *dest = *m +} + +// write writes the meta onto a page. +func (m *meta) write(p *page) { + if m.root.root >= m.pgid { + panic(fmt.Sprintf("root bucket pgid (%d) above high water mark (%d)", m.root.root, m.pgid)) + } else if m.freelist >= m.pgid { + panic(fmt.Sprintf("freelist pgid (%d) above high water mark (%d)", m.freelist, m.pgid)) + } + + // Page id is either going to be 0 or 1 which we can determine by the transaction ID. + p.id = pgid(m.txid % 2) + p.flags |= metaPageFlag + + // Calculate the checksum. + m.checksum = m.sum64() + + m.copy(p.meta()) +} + +// generates the checksum for the meta. +func (m *meta) sum64() uint64 { + var h = fnv.New64a() + _, _ = h.Write((*[unsafe.Offsetof(meta{}.checksum)]byte)(unsafe.Pointer(m))[:]) + return h.Sum64() +} + +// _assert will panic with a given formatted message if the given condition is false. +func _assert(condition bool, msg string, v ...interface{}) { + if !condition { + panic(fmt.Sprintf("assertion failed: "+msg, v...)) + } +} + +func warn(v ...interface{}) { fmt.Fprintln(os.Stderr, v...) } +func warnf(msg string, v ...interface{}) { fmt.Fprintf(os.Stderr, msg+"\n", v...) } + +func printstack() { + stack := strings.Join(strings.Split(string(debug.Stack()), "\n")[2:], "\n") + fmt.Fprintln(os.Stderr, stack) +} diff --git a/vendor/github.com/coreos/bbolt/db_test.go b/vendor/github.com/coreos/bbolt/db_test.go new file mode 100644 index 0000000..74ff93a --- /dev/null +++ b/vendor/github.com/coreos/bbolt/db_test.go @@ -0,0 +1,1706 @@ +package bolt_test + +import ( + "bytes" + "encoding/binary" + "errors" + "flag" + "fmt" + "hash/fnv" + "io/ioutil" + "log" + "os" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "sync" + "testing" + "time" + "unsafe" + + "github.com/boltdb/bolt" +) + +var statsFlag = flag.Bool("stats", false, "show performance stats") + +// version is the data file format version. +const version = 2 + +// magic is the marker value to indicate that a file is a Bolt DB. +const magic uint32 = 0xED0CDAED + +// pageSize is the size of one page in the data file. +const pageSize = 4096 + +// pageHeaderSize is the size of a page header. +const pageHeaderSize = 16 + +// meta represents a simplified version of a database meta page for testing. +type meta struct { + magic uint32 + version uint32 + _ uint32 + _ uint32 + _ [16]byte + _ uint64 + pgid uint64 + _ uint64 + checksum uint64 +} + +// Ensure that a database can be opened without error. +func TestOpen(t *testing.T) { + path := tempfile() + db, err := bolt.Open(path, 0666, nil) + if err != nil { + t.Fatal(err) + } else if db == nil { + t.Fatal("expected db") + } + + if s := db.Path(); s != path { + t.Fatalf("unexpected path: %s", s) + } + + if err := db.Close(); err != nil { + t.Fatal(err) + } +} + +// Ensure that opening a database with a blank path returns an error. +func TestOpen_ErrPathRequired(t *testing.T) { + _, err := bolt.Open("", 0666, nil) + if err == nil { + t.Fatalf("expected error") + } +} + +// Ensure that opening a database with a bad path returns an error. +func TestOpen_ErrNotExists(t *testing.T) { + _, err := bolt.Open(filepath.Join(tempfile(), "bad-path"), 0666, nil) + if err == nil { + t.Fatal("expected error") + } +} + +// Ensure that opening a file that is not a Bolt database returns ErrInvalid. +func TestOpen_ErrInvalid(t *testing.T) { + path := tempfile() + + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + if _, err := fmt.Fprintln(f, "this is not a bolt database"); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + defer os.Remove(path) + + if _, err := bolt.Open(path, 0666, nil); err != bolt.ErrInvalid { + t.Fatalf("unexpected error: %s", err) + } +} + +// Ensure that opening a file with two invalid versions returns ErrVersionMismatch. +func TestOpen_ErrVersionMismatch(t *testing.T) { + if pageSize != os.Getpagesize() { + t.Skip("page size mismatch") + } + + // Create empty database. + db := MustOpenDB() + path := db.Path() + defer db.MustClose() + + // Close database. + if err := db.DB.Close(); err != nil { + t.Fatal(err) + } + + // Read data file. + buf, err := ioutil.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + // Rewrite meta pages. + meta0 := (*meta)(unsafe.Pointer(&buf[pageHeaderSize])) + meta0.version++ + meta1 := (*meta)(unsafe.Pointer(&buf[pageSize+pageHeaderSize])) + meta1.version++ + if err := ioutil.WriteFile(path, buf, 0666); err != nil { + t.Fatal(err) + } + + // Reopen data file. + if _, err := bolt.Open(path, 0666, nil); err != bolt.ErrVersionMismatch { + t.Fatalf("unexpected error: %s", err) + } +} + +// Ensure that opening a file with two invalid checksums returns ErrChecksum. +func TestOpen_ErrChecksum(t *testing.T) { + if pageSize != os.Getpagesize() { + t.Skip("page size mismatch") + } + + // Create empty database. + db := MustOpenDB() + path := db.Path() + defer db.MustClose() + + // Close database. + if err := db.DB.Close(); err != nil { + t.Fatal(err) + } + + // Read data file. + buf, err := ioutil.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + // Rewrite meta pages. + meta0 := (*meta)(unsafe.Pointer(&buf[pageHeaderSize])) + meta0.pgid++ + meta1 := (*meta)(unsafe.Pointer(&buf[pageSize+pageHeaderSize])) + meta1.pgid++ + if err := ioutil.WriteFile(path, buf, 0666); err != nil { + t.Fatal(err) + } + + // Reopen data file. + if _, err := bolt.Open(path, 0666, nil); err != bolt.ErrChecksum { + t.Fatalf("unexpected error: %s", err) + } +} + +// Ensure that opening an already open database file will timeout. +func TestOpen_Timeout(t *testing.T) { + if runtime.GOOS == "solaris" { + t.Skip("solaris fcntl locks don't support intra-process locking") + } + + path := tempfile() + + // Open a data file. + db0, err := bolt.Open(path, 0666, nil) + if err != nil { + t.Fatal(err) + } else if db0 == nil { + t.Fatal("expected database") + } + + // Attempt to open the database again. + start := time.Now() + db1, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 100 * time.Millisecond}) + if err != bolt.ErrTimeout { + t.Fatalf("unexpected timeout: %s", err) + } else if db1 != nil { + t.Fatal("unexpected database") + } else if time.Since(start) <= 100*time.Millisecond { + t.Fatal("expected to wait at least timeout duration") + } + + if err := db0.Close(); err != nil { + t.Fatal(err) + } +} + +// Ensure that opening an already open database file will wait until its closed. +func TestOpen_Wait(t *testing.T) { + if runtime.GOOS == "solaris" { + t.Skip("solaris fcntl locks don't support intra-process locking") + } + + path := tempfile() + + // Open a data file. + db0, err := bolt.Open(path, 0666, nil) + if err != nil { + t.Fatal(err) + } + + // Close it in just a bit. + time.AfterFunc(100*time.Millisecond, func() { _ = db0.Close() }) + + // Attempt to open the database again. + start := time.Now() + db1, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 200 * time.Millisecond}) + if err != nil { + t.Fatal(err) + } else if time.Since(start) <= 100*time.Millisecond { + t.Fatal("expected to wait at least timeout duration") + } + + if err := db1.Close(); err != nil { + t.Fatal(err) + } +} + +// Ensure that opening a database does not increase its size. +// https://github.com/boltdb/bolt/issues/291 +func TestOpen_Size(t *testing.T) { + // Open a data file. + db := MustOpenDB() + path := db.Path() + defer db.MustClose() + + pagesize := db.Info().PageSize + + // Insert until we get above the minimum 4MB size. + if err := db.Update(func(tx *bolt.Tx) error { + b, _ := tx.CreateBucketIfNotExists([]byte("data")) + for i := 0; i < 10000; i++ { + if err := b.Put([]byte(fmt.Sprintf("%04d", i)), make([]byte, 1000)); err != nil { + t.Fatal(err) + } + } + return nil + }); err != nil { + t.Fatal(err) + } + + // Close database and grab the size. + if err := db.DB.Close(); err != nil { + t.Fatal(err) + } + sz := fileSize(path) + if sz == 0 { + t.Fatalf("unexpected new file size: %d", sz) + } + + // Reopen database, update, and check size again. + db0, err := bolt.Open(path, 0666, nil) + if err != nil { + t.Fatal(err) + } + if err := db0.Update(func(tx *bolt.Tx) error { + if err := tx.Bucket([]byte("data")).Put([]byte{0}, []byte{0}); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + if err := db0.Close(); err != nil { + t.Fatal(err) + } + newSz := fileSize(path) + if newSz == 0 { + t.Fatalf("unexpected new file size: %d", newSz) + } + + // Compare the original size with the new size. + // db size might increase by a few page sizes due to the new small update. + if sz < newSz-5*int64(pagesize) { + t.Fatalf("unexpected file growth: %d => %d", sz, newSz) + } +} + +// Ensure that opening a database beyond the max step size does not increase its size. +// https://github.com/boltdb/bolt/issues/303 +func TestOpen_Size_Large(t *testing.T) { + if testing.Short() { + t.Skip("short mode") + } + + // Open a data file. + db := MustOpenDB() + path := db.Path() + defer db.MustClose() + + pagesize := db.Info().PageSize + + // Insert until we get above the minimum 4MB size. + var index uint64 + for i := 0; i < 10000; i++ { + if err := db.Update(func(tx *bolt.Tx) error { + b, _ := tx.CreateBucketIfNotExists([]byte("data")) + for j := 0; j < 1000; j++ { + if err := b.Put(u64tob(index), make([]byte, 50)); err != nil { + t.Fatal(err) + } + index++ + } + return nil + }); err != nil { + t.Fatal(err) + } + } + + // Close database and grab the size. + if err := db.DB.Close(); err != nil { + t.Fatal(err) + } + sz := fileSize(path) + if sz == 0 { + t.Fatalf("unexpected new file size: %d", sz) + } else if sz < (1 << 30) { + t.Fatalf("expected larger initial size: %d", sz) + } + + // Reopen database, update, and check size again. + db0, err := bolt.Open(path, 0666, nil) + if err != nil { + t.Fatal(err) + } + if err := db0.Update(func(tx *bolt.Tx) error { + return tx.Bucket([]byte("data")).Put([]byte{0}, []byte{0}) + }); err != nil { + t.Fatal(err) + } + if err := db0.Close(); err != nil { + t.Fatal(err) + } + + newSz := fileSize(path) + if newSz == 0 { + t.Fatalf("unexpected new file size: %d", newSz) + } + + // Compare the original size with the new size. + // db size might increase by a few page sizes due to the new small update. + if sz < newSz-5*int64(pagesize) { + t.Fatalf("unexpected file growth: %d => %d", sz, newSz) + } +} + +// Ensure that a re-opened database is consistent. +func TestOpen_Check(t *testing.T) { + path := tempfile() + + db, err := bolt.Open(path, 0666, nil) + if err != nil { + t.Fatal(err) + } + if err := db.View(func(tx *bolt.Tx) error { return <-tx.Check() }); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + + db, err = bolt.Open(path, 0666, nil) + if err != nil { + t.Fatal(err) + } + if err := db.View(func(tx *bolt.Tx) error { return <-tx.Check() }); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } +} + +// Ensure that write errors to the meta file handler during initialization are returned. +func TestOpen_MetaInitWriteError(t *testing.T) { + t.Skip("pending") +} + +// Ensure that a database that is too small returns an error. +func TestOpen_FileTooSmall(t *testing.T) { + path := tempfile() + + db, err := bolt.Open(path, 0666, nil) + if err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + + // corrupt the database + if err := os.Truncate(path, int64(os.Getpagesize())); err != nil { + t.Fatal(err) + } + + db, err = bolt.Open(path, 0666, nil) + if err == nil || err.Error() != "file size too small" { + t.Fatalf("unexpected error: %s", err) + } +} + +// Ensure that a database can be opened in read-only mode by multiple processes +// and that a database can not be opened in read-write mode and in read-only +// mode at the same time. +func TestOpen_ReadOnly(t *testing.T) { + if runtime.GOOS == "solaris" { + t.Skip("solaris fcntl locks don't support intra-process locking") + } + + bucket, key, value := []byte(`bucket`), []byte(`key`), []byte(`value`) + + path := tempfile() + + // Open in read-write mode. + db, err := bolt.Open(path, 0666, nil) + if err != nil { + t.Fatal(err) + } else if db.IsReadOnly() { + t.Fatal("db should not be in read only mode") + } + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket(bucket) + if err != nil { + return err + } + if err := b.Put(key, value); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + + // Open in read-only mode. + db0, err := bolt.Open(path, 0666, &bolt.Options{ReadOnly: true}) + if err != nil { + t.Fatal(err) + } + + // Opening in read-write mode should return an error. + if _, err = bolt.Open(path, 0666, &bolt.Options{Timeout: time.Millisecond * 100}); err == nil { + t.Fatal("expected error") + } + + // And again (in read-only mode). + db1, err := bolt.Open(path, 0666, &bolt.Options{ReadOnly: true}) + if err != nil { + t.Fatal(err) + } + + // Verify both read-only databases are accessible. + for _, db := range []*bolt.DB{db0, db1} { + // Verify is is in read only mode indeed. + if !db.IsReadOnly() { + t.Fatal("expected read only mode") + } + + // Read-only databases should not allow updates. + if err := db.Update(func(*bolt.Tx) error { + panic(`should never get here`) + }); err != bolt.ErrDatabaseReadOnly { + t.Fatalf("unexpected error: %s", err) + } + + // Read-only databases should not allow beginning writable txns. + if _, err := db.Begin(true); err != bolt.ErrDatabaseReadOnly { + t.Fatalf("unexpected error: %s", err) + } + + // Verify the data. + if err := db.View(func(tx *bolt.Tx) error { + b := tx.Bucket(bucket) + if b == nil { + return fmt.Errorf("expected bucket `%s`", string(bucket)) + } + + got := string(b.Get(key)) + expected := string(value) + if got != expected { + return fmt.Errorf("expected `%s`, got `%s`", expected, got) + } + return nil + }); err != nil { + t.Fatal(err) + } + } + + if err := db0.Close(); err != nil { + t.Fatal(err) + } + if err := db1.Close(); err != nil { + t.Fatal(err) + } +} + +// TestDB_Open_InitialMmapSize tests if having InitialMmapSize large enough +// to hold data from concurrent write transaction resolves the issue that +// read transaction blocks the write transaction and causes deadlock. +// This is a very hacky test since the mmap size is not exposed. +func TestDB_Open_InitialMmapSize(t *testing.T) { + path := tempfile() + defer os.Remove(path) + + initMmapSize := 1 << 31 // 2GB + testWriteSize := 1 << 27 // 134MB + + db, err := bolt.Open(path, 0666, &bolt.Options{InitialMmapSize: initMmapSize}) + if err != nil { + t.Fatal(err) + } + + // create a long-running read transaction + // that never gets closed while writing + rtx, err := db.Begin(false) + if err != nil { + t.Fatal(err) + } + + // create a write transaction + wtx, err := db.Begin(true) + if err != nil { + t.Fatal(err) + } + + b, err := wtx.CreateBucket([]byte("test")) + if err != nil { + t.Fatal(err) + } + + // and commit a large write + err = b.Put([]byte("foo"), make([]byte, testWriteSize)) + if err != nil { + t.Fatal(err) + } + + done := make(chan struct{}) + + go func() { + if err := wtx.Commit(); err != nil { + t.Fatal(err) + } + done <- struct{}{} + }() + + select { + case <-time.After(5 * time.Second): + t.Errorf("unexpected that the reader blocks writer") + case <-done: + } + + if err := rtx.Rollback(); err != nil { + t.Fatal(err) + } +} + +// Ensure that a database cannot open a transaction when it's not open. +func TestDB_Begin_ErrDatabaseNotOpen(t *testing.T) { + var db bolt.DB + if _, err := db.Begin(false); err != bolt.ErrDatabaseNotOpen { + t.Fatalf("unexpected error: %s", err) + } +} + +// Ensure that a read-write transaction can be retrieved. +func TestDB_BeginRW(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + tx, err := db.Begin(true) + if err != nil { + t.Fatal(err) + } else if tx == nil { + t.Fatal("expected tx") + } + + if tx.DB() != db.DB { + t.Fatal("unexpected tx database") + } else if !tx.Writable() { + t.Fatal("expected writable tx") + } + + if err := tx.Commit(); err != nil { + t.Fatal(err) + } +} + +// Ensure that opening a transaction while the DB is closed returns an error. +func TestDB_BeginRW_Closed(t *testing.T) { + var db bolt.DB + if _, err := db.Begin(true); err != bolt.ErrDatabaseNotOpen { + t.Fatalf("unexpected error: %s", err) + } +} + +func TestDB_Close_PendingTx_RW(t *testing.T) { testDB_Close_PendingTx(t, true) } +func TestDB_Close_PendingTx_RO(t *testing.T) { testDB_Close_PendingTx(t, false) } + +// Ensure that a database cannot close while transactions are open. +func testDB_Close_PendingTx(t *testing.T, writable bool) { + db := MustOpenDB() + defer db.MustClose() + + // Start transaction. + tx, err := db.Begin(true) + if err != nil { + t.Fatal(err) + } + + // Open update in separate goroutine. + done := make(chan struct{}) + go func() { + if err := db.Close(); err != nil { + t.Fatal(err) + } + close(done) + }() + + // Ensure database hasn't closed. + time.Sleep(100 * time.Millisecond) + select { + case <-done: + t.Fatal("database closed too early") + default: + } + + // Commit transaction. + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + + // Ensure database closed now. + time.Sleep(100 * time.Millisecond) + select { + case <-done: + default: + t.Fatal("database did not close") + } +} + +// Ensure a database can provide a transactional block. +func TestDB_Update(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("baz"), []byte("bat")); err != nil { + t.Fatal(err) + } + if err := b.Delete([]byte("foo")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + if err := db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("widgets")) + if v := b.Get([]byte("foo")); v != nil { + t.Fatalf("expected nil value, got: %v", v) + } + if v := b.Get([]byte("baz")); !bytes.Equal(v, []byte("bat")) { + t.Fatalf("unexpected value: %v", v) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure a closed database returns an error while running a transaction block +func TestDB_Update_Closed(t *testing.T) { + var db bolt.DB + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + return nil + }); err != bolt.ErrDatabaseNotOpen { + t.Fatalf("unexpected error: %s", err) + } +} + +// Ensure a panic occurs while trying to commit a managed transaction. +func TestDB_Update_ManualCommit(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + var panicked bool + if err := db.Update(func(tx *bolt.Tx) error { + func() { + defer func() { + if r := recover(); r != nil { + panicked = true + } + }() + + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + }() + return nil + }); err != nil { + t.Fatal(err) + } else if !panicked { + t.Fatal("expected panic") + } +} + +// Ensure a panic occurs while trying to rollback a managed transaction. +func TestDB_Update_ManualRollback(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + var panicked bool + if err := db.Update(func(tx *bolt.Tx) error { + func() { + defer func() { + if r := recover(); r != nil { + panicked = true + } + }() + + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + }() + return nil + }); err != nil { + t.Fatal(err) + } else if !panicked { + t.Fatal("expected panic") + } +} + +// Ensure a panic occurs while trying to commit a managed transaction. +func TestDB_View_ManualCommit(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + var panicked bool + if err := db.View(func(tx *bolt.Tx) error { + func() { + defer func() { + if r := recover(); r != nil { + panicked = true + } + }() + + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + }() + return nil + }); err != nil { + t.Fatal(err) + } else if !panicked { + t.Fatal("expected panic") + } +} + +// Ensure a panic occurs while trying to rollback a managed transaction. +func TestDB_View_ManualRollback(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + var panicked bool + if err := db.View(func(tx *bolt.Tx) error { + func() { + defer func() { + if r := recover(); r != nil { + panicked = true + } + }() + + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + }() + return nil + }); err != nil { + t.Fatal(err) + } else if !panicked { + t.Fatal("expected panic") + } +} + +// Ensure a write transaction that panics does not hold open locks. +func TestDB_Update_Panic(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + // Panic during update but recover. + func() { + defer func() { + if r := recover(); r != nil { + t.Log("recover: update", r) + } + }() + + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + panic("omg") + }); err != nil { + t.Fatal(err) + } + }() + + // Verify we can update again. + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + // Verify that our change persisted. + if err := db.Update(func(tx *bolt.Tx) error { + if tx.Bucket([]byte("widgets")) == nil { + t.Fatal("expected bucket") + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure a database can return an error through a read-only transactional block. +func TestDB_View_Error(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.View(func(tx *bolt.Tx) error { + return errors.New("xxx") + }); err == nil || err.Error() != "xxx" { + t.Fatalf("unexpected error: %s", err) + } +} + +// Ensure a read transaction that panics does not hold open locks. +func TestDB_View_Panic(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + // Panic during view transaction but recover. + func() { + defer func() { + if r := recover(); r != nil { + t.Log("recover: view", r) + } + }() + + if err := db.View(func(tx *bolt.Tx) error { + if tx.Bucket([]byte("widgets")) == nil { + t.Fatal("expected bucket") + } + panic("omg") + }); err != nil { + t.Fatal(err) + } + }() + + // Verify that we can still use read transactions. + if err := db.View(func(tx *bolt.Tx) error { + if tx.Bucket([]byte("widgets")) == nil { + t.Fatal("expected bucket") + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that DB stats can be returned. +func TestDB_Stats(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucket([]byte("widgets")) + return err + }); err != nil { + t.Fatal(err) + } + + stats := db.Stats() + if stats.TxStats.PageCount != 2 { + t.Fatalf("unexpected TxStats.PageCount: %d", stats.TxStats.PageCount) + } else if stats.FreePageN != 0 { + t.Fatalf("unexpected FreePageN != 0: %d", stats.FreePageN) + } else if stats.PendingPageN != 2 { + t.Fatalf("unexpected PendingPageN != 2: %d", stats.PendingPageN) + } +} + +// Ensure that database pages are in expected order and type. +func TestDB_Consistency(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucket([]byte("widgets")) + return err + }); err != nil { + t.Fatal(err) + } + + for i := 0; i < 10; i++ { + if err := db.Update(func(tx *bolt.Tx) error { + if err := tx.Bucket([]byte("widgets")).Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + } + + if err := db.Update(func(tx *bolt.Tx) error { + if p, _ := tx.Page(0); p == nil { + t.Fatal("expected page") + } else if p.Type != "meta" { + t.Fatalf("unexpected page type: %s", p.Type) + } + + if p, _ := tx.Page(1); p == nil { + t.Fatal("expected page") + } else if p.Type != "meta" { + t.Fatalf("unexpected page type: %s", p.Type) + } + + if p, _ := tx.Page(2); p == nil { + t.Fatal("expected page") + } else if p.Type != "free" { + t.Fatalf("unexpected page type: %s", p.Type) + } + + if p, _ := tx.Page(3); p == nil { + t.Fatal("expected page") + } else if p.Type != "free" { + t.Fatalf("unexpected page type: %s", p.Type) + } + + if p, _ := tx.Page(4); p == nil { + t.Fatal("expected page") + } else if p.Type != "leaf" { + t.Fatalf("unexpected page type: %s", p.Type) + } + + if p, _ := tx.Page(5); p == nil { + t.Fatal("expected page") + } else if p.Type != "freelist" { + t.Fatalf("unexpected page type: %s", p.Type) + } + + if p, _ := tx.Page(6); p != nil { + t.Fatal("unexpected page") + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that DB stats can be subtracted from one another. +func TestDBStats_Sub(t *testing.T) { + var a, b bolt.Stats + a.TxStats.PageCount = 3 + a.FreePageN = 4 + b.TxStats.PageCount = 10 + b.FreePageN = 14 + diff := b.Sub(&a) + if diff.TxStats.PageCount != 7 { + t.Fatalf("unexpected TxStats.PageCount: %d", diff.TxStats.PageCount) + } + + // free page stats are copied from the receiver and not subtracted + if diff.FreePageN != 14 { + t.Fatalf("unexpected FreePageN: %d", diff.FreePageN) + } +} + +// Ensure two functions can perform updates in a single batch. +func TestDB_Batch(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + // Iterate over multiple updates in separate goroutines. + n := 2 + ch := make(chan error) + for i := 0; i < n; i++ { + go func(i int) { + ch <- db.Batch(func(tx *bolt.Tx) error { + return tx.Bucket([]byte("widgets")).Put(u64tob(uint64(i)), []byte{}) + }) + }(i) + } + + // Check all responses to make sure there's no error. + for i := 0; i < n; i++ { + if err := <-ch; err != nil { + t.Fatal(err) + } + } + + // Ensure data is correct. + if err := db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("widgets")) + for i := 0; i < n; i++ { + if v := b.Get(u64tob(uint64(i))); v == nil { + t.Errorf("key not found: %d", i) + } + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +func TestDB_Batch_Panic(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + var sentinel int + var bork = &sentinel + var problem interface{} + var err error + + // Execute a function inside a batch that panics. + func() { + defer func() { + if p := recover(); p != nil { + problem = p + } + }() + err = db.Batch(func(tx *bolt.Tx) error { + panic(bork) + }) + }() + + // Verify there is no error. + if g, e := err, error(nil); g != e { + t.Fatalf("wrong error: %v != %v", g, e) + } + // Verify the panic was captured. + if g, e := problem, bork; g != e { + t.Fatalf("wrong error: %v != %v", g, e) + } +} + +func TestDB_BatchFull(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucket([]byte("widgets")) + return err + }); err != nil { + t.Fatal(err) + } + + const size = 3 + // buffered so we never leak goroutines + ch := make(chan error, size) + put := func(i int) { + ch <- db.Batch(func(tx *bolt.Tx) error { + return tx.Bucket([]byte("widgets")).Put(u64tob(uint64(i)), []byte{}) + }) + } + + db.MaxBatchSize = size + // high enough to never trigger here + db.MaxBatchDelay = 1 * time.Hour + + go put(1) + go put(2) + + // Give the batch a chance to exhibit bugs. + time.Sleep(10 * time.Millisecond) + + // not triggered yet + select { + case <-ch: + t.Fatalf("batch triggered too early") + default: + } + + go put(3) + + // Check all responses to make sure there's no error. + for i := 0; i < size; i++ { + if err := <-ch; err != nil { + t.Fatal(err) + } + } + + // Ensure data is correct. + if err := db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("widgets")) + for i := 1; i <= size; i++ { + if v := b.Get(u64tob(uint64(i))); v == nil { + t.Errorf("key not found: %d", i) + } + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +func TestDB_BatchTime(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucket([]byte("widgets")) + return err + }); err != nil { + t.Fatal(err) + } + + const size = 1 + // buffered so we never leak goroutines + ch := make(chan error, size) + put := func(i int) { + ch <- db.Batch(func(tx *bolt.Tx) error { + return tx.Bucket([]byte("widgets")).Put(u64tob(uint64(i)), []byte{}) + }) + } + + db.MaxBatchSize = 1000 + db.MaxBatchDelay = 0 + + go put(1) + + // Batch must trigger by time alone. + + // Check all responses to make sure there's no error. + for i := 0; i < size; i++ { + if err := <-ch; err != nil { + t.Fatal(err) + } + } + + // Ensure data is correct. + if err := db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("widgets")) + for i := 1; i <= size; i++ { + if v := b.Get(u64tob(uint64(i))); v == nil { + t.Errorf("key not found: %d", i) + } + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +func ExampleDB_Update() { + // Open the database. + db, err := bolt.Open(tempfile(), 0666, nil) + if err != nil { + log.Fatal(err) + } + defer os.Remove(db.Path()) + + // Execute several commands within a read-write transaction. + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + return err + } + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + return err + } + return nil + }); err != nil { + log.Fatal(err) + } + + // Read the value back from a separate read-only transaction. + if err := db.View(func(tx *bolt.Tx) error { + value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) + fmt.Printf("The value of 'foo' is: %s\n", value) + return nil + }); err != nil { + log.Fatal(err) + } + + // Close database to release the file lock. + if err := db.Close(); err != nil { + log.Fatal(err) + } + + // Output: + // The value of 'foo' is: bar +} + +func ExampleDB_View() { + // Open the database. + db, err := bolt.Open(tempfile(), 0666, nil) + if err != nil { + log.Fatal(err) + } + defer os.Remove(db.Path()) + + // Insert data into a bucket. + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("people")) + if err != nil { + return err + } + if err := b.Put([]byte("john"), []byte("doe")); err != nil { + return err + } + if err := b.Put([]byte("susy"), []byte("que")); err != nil { + return err + } + return nil + }); err != nil { + log.Fatal(err) + } + + // Access data from within a read-only transactional block. + if err := db.View(func(tx *bolt.Tx) error { + v := tx.Bucket([]byte("people")).Get([]byte("john")) + fmt.Printf("John's last name is %s.\n", v) + return nil + }); err != nil { + log.Fatal(err) + } + + // Close database to release the file lock. + if err := db.Close(); err != nil { + log.Fatal(err) + } + + // Output: + // John's last name is doe. +} + +func ExampleDB_Begin_ReadOnly() { + // Open the database. + db, err := bolt.Open(tempfile(), 0666, nil) + if err != nil { + log.Fatal(err) + } + defer os.Remove(db.Path()) + + // Create a bucket using a read-write transaction. + if err := db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucket([]byte("widgets")) + return err + }); err != nil { + log.Fatal(err) + } + + // Create several keys in a transaction. + tx, err := db.Begin(true) + if err != nil { + log.Fatal(err) + } + b := tx.Bucket([]byte("widgets")) + if err := b.Put([]byte("john"), []byte("blue")); err != nil { + log.Fatal(err) + } + if err := b.Put([]byte("abby"), []byte("red")); err != nil { + log.Fatal(err) + } + if err := b.Put([]byte("zephyr"), []byte("purple")); err != nil { + log.Fatal(err) + } + if err := tx.Commit(); err != nil { + log.Fatal(err) + } + + // Iterate over the values in sorted key order. + tx, err = db.Begin(false) + if err != nil { + log.Fatal(err) + } + c := tx.Bucket([]byte("widgets")).Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + fmt.Printf("%s likes %s\n", k, v) + } + + if err := tx.Rollback(); err != nil { + log.Fatal(err) + } + + if err := db.Close(); err != nil { + log.Fatal(err) + } + + // Output: + // abby likes red + // john likes blue + // zephyr likes purple +} + +func BenchmarkDBBatchAutomatic(b *testing.B) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucket([]byte("bench")) + return err + }); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + start := make(chan struct{}) + var wg sync.WaitGroup + + for round := 0; round < 1000; round++ { + wg.Add(1) + + go func(id uint32) { + defer wg.Done() + <-start + + h := fnv.New32a() + buf := make([]byte, 4) + binary.LittleEndian.PutUint32(buf, id) + _, _ = h.Write(buf[:]) + k := h.Sum(nil) + insert := func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("bench")) + return b.Put(k, []byte("filler")) + } + if err := db.Batch(insert); err != nil { + b.Error(err) + return + } + }(uint32(round)) + } + close(start) + wg.Wait() + } + + b.StopTimer() + validateBatchBench(b, db) +} + +func BenchmarkDBBatchSingle(b *testing.B) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucket([]byte("bench")) + return err + }); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + start := make(chan struct{}) + var wg sync.WaitGroup + + for round := 0; round < 1000; round++ { + wg.Add(1) + go func(id uint32) { + defer wg.Done() + <-start + + h := fnv.New32a() + buf := make([]byte, 4) + binary.LittleEndian.PutUint32(buf, id) + _, _ = h.Write(buf[:]) + k := h.Sum(nil) + insert := func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("bench")) + return b.Put(k, []byte("filler")) + } + if err := db.Update(insert); err != nil { + b.Error(err) + return + } + }(uint32(round)) + } + close(start) + wg.Wait() + } + + b.StopTimer() + validateBatchBench(b, db) +} + +func BenchmarkDBBatchManual10x100(b *testing.B) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucket([]byte("bench")) + return err + }); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + start := make(chan struct{}) + var wg sync.WaitGroup + + for major := 0; major < 10; major++ { + wg.Add(1) + go func(id uint32) { + defer wg.Done() + <-start + + insert100 := func(tx *bolt.Tx) error { + h := fnv.New32a() + buf := make([]byte, 4) + for minor := uint32(0); minor < 100; minor++ { + binary.LittleEndian.PutUint32(buf, uint32(id*100+minor)) + h.Reset() + _, _ = h.Write(buf[:]) + k := h.Sum(nil) + b := tx.Bucket([]byte("bench")) + if err := b.Put(k, []byte("filler")); err != nil { + return err + } + } + return nil + } + if err := db.Update(insert100); err != nil { + b.Fatal(err) + } + }(uint32(major)) + } + close(start) + wg.Wait() + } + + b.StopTimer() + validateBatchBench(b, db) +} + +func validateBatchBench(b *testing.B, db *DB) { + var rollback = errors.New("sentinel error to cause rollback") + validate := func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte("bench")) + h := fnv.New32a() + buf := make([]byte, 4) + for id := uint32(0); id < 1000; id++ { + binary.LittleEndian.PutUint32(buf, id) + h.Reset() + _, _ = h.Write(buf[:]) + k := h.Sum(nil) + v := bucket.Get(k) + if v == nil { + b.Errorf("not found id=%d key=%x", id, k) + continue + } + if g, e := v, []byte("filler"); !bytes.Equal(g, e) { + b.Errorf("bad value for id=%d key=%x: %s != %q", id, k, g, e) + } + if err := bucket.Delete(k); err != nil { + return err + } + } + // should be empty now + c := bucket.Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + b.Errorf("unexpected key: %x = %q", k, v) + } + return rollback + } + if err := db.Update(validate); err != nil && err != rollback { + b.Error(err) + } +} + +// DB is a test wrapper for bolt.DB. +type DB struct { + *bolt.DB +} + +// MustOpenDB returns a new, open DB at a temporary location. +func MustOpenDB() *DB { + db, err := bolt.Open(tempfile(), 0666, nil) + if err != nil { + panic(err) + } + return &DB{db} +} + +// Close closes the database and deletes the underlying file. +func (db *DB) Close() error { + // Log statistics. + if *statsFlag { + db.PrintStats() + } + + // Check database consistency after every test. + db.MustCheck() + + // Close database and remove file. + defer os.Remove(db.Path()) + return db.DB.Close() +} + +// MustClose closes the database and deletes the underlying file. Panic on error. +func (db *DB) MustClose() { + if err := db.Close(); err != nil { + panic(err) + } +} + +// PrintStats prints the database stats +func (db *DB) PrintStats() { + var stats = db.Stats() + fmt.Printf("[db] %-20s %-20s %-20s\n", + fmt.Sprintf("pg(%d/%d)", stats.TxStats.PageCount, stats.TxStats.PageAlloc), + fmt.Sprintf("cur(%d)", stats.TxStats.CursorCount), + fmt.Sprintf("node(%d/%d)", stats.TxStats.NodeCount, stats.TxStats.NodeDeref), + ) + fmt.Printf(" %-20s %-20s %-20s\n", + fmt.Sprintf("rebal(%d/%v)", stats.TxStats.Rebalance, truncDuration(stats.TxStats.RebalanceTime)), + fmt.Sprintf("spill(%d/%v)", stats.TxStats.Spill, truncDuration(stats.TxStats.SpillTime)), + fmt.Sprintf("w(%d/%v)", stats.TxStats.Write, truncDuration(stats.TxStats.WriteTime)), + ) +} + +// MustCheck runs a consistency check on the database and panics if any errors are found. +func (db *DB) MustCheck() { + if err := db.Update(func(tx *bolt.Tx) error { + // Collect all the errors. + var errors []error + for err := range tx.Check() { + errors = append(errors, err) + if len(errors) > 10 { + break + } + } + + // If errors occurred, copy the DB and print the errors. + if len(errors) > 0 { + var path = tempfile() + if err := tx.CopyFile(path, 0600); err != nil { + panic(err) + } + + // Print errors. + fmt.Print("\n\n") + fmt.Printf("consistency check failed (%d errors)\n", len(errors)) + for _, err := range errors { + fmt.Println(err) + } + fmt.Println("") + fmt.Println("db saved to:") + fmt.Println(path) + fmt.Print("\n\n") + os.Exit(-1) + } + + return nil + }); err != nil && err != bolt.ErrDatabaseNotOpen { + panic(err) + } +} + +// CopyTempFile copies a database to a temporary file. +func (db *DB) CopyTempFile() { + path := tempfile() + if err := db.View(func(tx *bolt.Tx) error { + return tx.CopyFile(path, 0600) + }); err != nil { + panic(err) + } + fmt.Println("db copied to: ", path) +} + +// tempfile returns a temporary file path. +func tempfile() string { + f, err := ioutil.TempFile("", "bolt-") + if err != nil { + panic(err) + } + if err := f.Close(); err != nil { + panic(err) + } + if err := os.Remove(f.Name()); err != nil { + panic(err) + } + return f.Name() +} + +// mustContainKeys checks that a bucket contains a given set of keys. +func mustContainKeys(b *bolt.Bucket, m map[string]string) { + found := make(map[string]string) + if err := b.ForEach(func(k, _ []byte) error { + found[string(k)] = "" + return nil + }); err != nil { + panic(err) + } + + // Check for keys found in bucket that shouldn't be there. + var keys []string + for k, _ := range found { + if _, ok := m[string(k)]; !ok { + keys = append(keys, k) + } + } + if len(keys) > 0 { + sort.Strings(keys) + panic(fmt.Sprintf("keys found(%d): %s", len(keys), strings.Join(keys, ","))) + } + + // Check for keys not found in bucket that should be there. + for k, _ := range m { + if _, ok := found[string(k)]; !ok { + keys = append(keys, k) + } + } + if len(keys) > 0 { + sort.Strings(keys) + panic(fmt.Sprintf("keys not found(%d): %s", len(keys), strings.Join(keys, ","))) + } +} + +func trunc(b []byte, length int) []byte { + if length < len(b) { + return b[:length] + } + return b +} + +func truncDuration(d time.Duration) string { + return regexp.MustCompile(`^(\d+)(\.\d+)`).ReplaceAllString(d.String(), "$1") +} + +func fileSize(path string) int64 { + fi, err := os.Stat(path) + if err != nil { + return 0 + } + return fi.Size() +} + +func warn(v ...interface{}) { fmt.Fprintln(os.Stderr, v...) } +func warnf(msg string, v ...interface{}) { fmt.Fprintf(os.Stderr, msg+"\n", v...) } + +// u64tob converts a uint64 into an 8-byte slice. +func u64tob(v uint64) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, v) + return b +} + +// btou64 converts an 8-byte slice into an uint64. +func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) } diff --git a/vendor/github.com/coreos/bbolt/doc.go b/vendor/github.com/coreos/bbolt/doc.go new file mode 100644 index 0000000..cc93784 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/doc.go @@ -0,0 +1,44 @@ +/* +Package bolt implements a low-level key/value store in pure Go. It supports +fully serializable transactions, ACID semantics, and lock-free MVCC with +multiple readers and a single writer. Bolt can be used for projects that +want a simple data store without the need to add large dependencies such as +Postgres or MySQL. + +Bolt is a single-level, zero-copy, B+tree data store. This means that Bolt is +optimized for fast read access and does not require recovery in the event of a +system crash. Transactions which have not finished committing will simply be +rolled back in the event of a crash. + +The design of Bolt is based on Howard Chu's LMDB database project. + +Bolt currently works on Windows, Mac OS X, and Linux. + + +Basics + +There are only a few types in Bolt: DB, Bucket, Tx, and Cursor. The DB is +a collection of buckets and is represented by a single file on disk. A bucket is +a collection of unique keys that are associated with values. + +Transactions provide either read-only or read-write access to the database. +Read-only transactions can retrieve key/value pairs and can use Cursors to +iterate over the dataset sequentially. Read-write transactions can create and +delete buckets and can insert and remove keys. Only one read-write transaction +is allowed at a time. + + +Caveats + +The database uses a read-only, memory-mapped data file to ensure that +applications cannot corrupt the database, however, this means that keys and +values returned from Bolt cannot be changed. Writing to a read-only byte slice +will cause Go to panic. + +Keys and values retrieved from the database are only valid for the life of +the transaction. When used outside the transaction, these byte slices can +point to different data or can point to invalid memory which will cause a panic. + + +*/ +package bolt diff --git a/vendor/github.com/coreos/bbolt/errors.go b/vendor/github.com/coreos/bbolt/errors.go new file mode 100644 index 0000000..a3620a3 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/errors.go @@ -0,0 +1,71 @@ +package bolt + +import "errors" + +// These errors can be returned when opening or calling methods on a DB. +var ( + // ErrDatabaseNotOpen is returned when a DB instance is accessed before it + // is opened or after it is closed. + ErrDatabaseNotOpen = errors.New("database not open") + + // ErrDatabaseOpen is returned when opening a database that is + // already open. + ErrDatabaseOpen = errors.New("database already open") + + // ErrInvalid is returned when both meta pages on a database are invalid. + // This typically occurs when a file is not a bolt database. + ErrInvalid = errors.New("invalid database") + + // ErrVersionMismatch is returned when the data file was created with a + // different version of Bolt. + ErrVersionMismatch = errors.New("version mismatch") + + // ErrChecksum is returned when either meta page checksum does not match. + ErrChecksum = errors.New("checksum error") + + // ErrTimeout is returned when a database cannot obtain an exclusive lock + // on the data file after the timeout passed to Open(). + ErrTimeout = errors.New("timeout") +) + +// These errors can occur when beginning or committing a Tx. +var ( + // ErrTxNotWritable is returned when performing a write operation on a + // read-only transaction. + ErrTxNotWritable = errors.New("tx not writable") + + // ErrTxClosed is returned when committing or rolling back a transaction + // that has already been committed or rolled back. + ErrTxClosed = errors.New("tx closed") + + // ErrDatabaseReadOnly is returned when a mutating transaction is started on a + // read-only database. + ErrDatabaseReadOnly = errors.New("database is in read-only mode") +) + +// These errors can occur when putting or deleting a value or a bucket. +var ( + // ErrBucketNotFound is returned when trying to access a bucket that has + // not been created yet. + ErrBucketNotFound = errors.New("bucket not found") + + // ErrBucketExists is returned when creating a bucket that already exists. + ErrBucketExists = errors.New("bucket already exists") + + // ErrBucketNameRequired is returned when creating a bucket with a blank name. + ErrBucketNameRequired = errors.New("bucket name required") + + // ErrKeyRequired is returned when inserting a zero-length key. + ErrKeyRequired = errors.New("key required") + + // ErrKeyTooLarge is returned when inserting a key that is larger than MaxKeySize. + ErrKeyTooLarge = errors.New("key too large") + + // ErrValueTooLarge is returned when inserting a value that is larger than MaxValueSize. + ErrValueTooLarge = errors.New("value too large") + + // ErrIncompatibleValue is returned when trying create or delete a bucket + // on an existing non-bucket key or when trying to create or delete a + // non-bucket key on an existing bucket key. + ErrIncompatibleValue = errors.New("incompatible value") +) diff --git a/vendor/github.com/coreos/bbolt/freelist.go b/vendor/github.com/coreos/bbolt/freelist.go new file mode 100644 index 0000000..1b7ba91 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/freelist.go @@ -0,0 +1,248 @@ +package bolt + +import ( + "fmt" + "sort" + "unsafe" +) + +// freelist represents a list of all pages that are available for allocation. +// It also tracks pages that have been freed but are still in use by open transactions. +type freelist struct { + ids []pgid // all free and available free page ids. + pending map[txid][]pgid // mapping of soon-to-be free page ids by tx. + cache map[pgid]bool // fast lookup of all free and pending page ids. +} + +// newFreelist returns an empty, initialized freelist. +func newFreelist() *freelist { + return &freelist{ + pending: make(map[txid][]pgid), + cache: make(map[pgid]bool), + } +} + +// size returns the size of the page after serialization. +func (f *freelist) size() int { + return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * f.count()) +} + +// count returns count of pages on the freelist +func (f *freelist) count() int { + return f.free_count() + f.pending_count() +} + +// free_count returns count of free pages +func (f *freelist) free_count() int { + return len(f.ids) +} + +// pending_count returns count of pending pages +func (f *freelist) pending_count() int { + var count int + for _, list := range f.pending { + count += len(list) + } + return count +} + +// all returns a list of all free ids and all pending ids in one sorted list. +func (f *freelist) all() []pgid { + m := make(pgids, 0) + + for _, list := range f.pending { + m = append(m, list...) + } + + sort.Sort(m) + return pgids(f.ids).merge(m) +} + +// allocate returns the starting page id of a contiguous list of pages of a given size. +// If a contiguous block cannot be found then 0 is returned. +func (f *freelist) allocate(n int) pgid { + if len(f.ids) == 0 { + return 0 + } + + var initial, previd pgid + for i, id := range f.ids { + if id <= 1 { + panic(fmt.Sprintf("invalid page allocation: %d", id)) + } + + // Reset initial page if this is not contiguous. + if previd == 0 || id-previd != 1 { + initial = id + } + + // If we found a contiguous block then remove it and return it. + if (id-initial)+1 == pgid(n) { + // If we're allocating off the beginning then take the fast path + // and just adjust the existing slice. This will use extra memory + // temporarily but the append() in free() will realloc the slice + // as is necessary. + if (i + 1) == n { + f.ids = f.ids[i+1:] + } else { + copy(f.ids[i-n+1:], f.ids[i+1:]) + f.ids = f.ids[:len(f.ids)-n] + } + + // Remove from the free cache. + for i := pgid(0); i < pgid(n); i++ { + delete(f.cache, initial+i) + } + + return initial + } + + previd = id + } + return 0 +} + +// free releases a page and its overflow for a given transaction id. +// If the page is already free then a panic will occur. +func (f *freelist) free(txid txid, p *page) { + if p.id <= 1 { + panic(fmt.Sprintf("cannot free page 0 or 1: %d", p.id)) + } + + // Free page and all its overflow pages. + var ids = f.pending[txid] + for id := p.id; id <= p.id+pgid(p.overflow); id++ { + // Verify that page is not already free. + if f.cache[id] { + panic(fmt.Sprintf("page %d already freed", id)) + } + + // Add to the freelist and cache. + ids = append(ids, id) + f.cache[id] = true + } + f.pending[txid] = ids +} + +// release moves all page ids for a transaction id (or older) to the freelist. +func (f *freelist) release(txid txid) { + m := make(pgids, 0) + for tid, ids := range f.pending { + if tid <= txid { + // Move transaction's pending pages to the available freelist. + // Don't remove from the cache since the page is still free. + m = append(m, ids...) + delete(f.pending, tid) + } + } + sort.Sort(m) + f.ids = pgids(f.ids).merge(m) +} + +// rollback removes the pages from a given pending tx. +func (f *freelist) rollback(txid txid) { + // Remove page ids from cache. + for _, id := range f.pending[txid] { + delete(f.cache, id) + } + + // Remove pages from pending list. + delete(f.pending, txid) +} + +// freed returns whether a given page is in the free list. +func (f *freelist) freed(pgid pgid) bool { + return f.cache[pgid] +} + +// read initializes the freelist from a freelist page. +func (f *freelist) read(p *page) { + // If the page.count is at the max uint16 value (64k) then it's considered + // an overflow and the size of the freelist is stored as the first element. + idx, count := 0, int(p.count) + if count == 0xFFFF { + idx = 1 + count = int(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0]) + } + + // Copy the list of page ids from the freelist. + if count == 0 { + f.ids = nil + } else { + ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count] + f.ids = make([]pgid, len(ids)) + copy(f.ids, ids) + + // Make sure they're sorted. + sort.Sort(pgids(f.ids)) + } + + // Rebuild the page cache. + f.reindex() +} + +// write writes the page ids onto a freelist page. All free and pending ids are +// saved to disk since in the event of a program crash, all pending ids will +// become free. +func (f *freelist) write(p *page) error { + // Combine the old free pgids and pgids waiting on an open transaction. + ids := f.all() + + // Update the header flag. + p.flags |= freelistPageFlag + + // The page.count can only hold up to 64k elements so if we overflow that + // number then we handle it by putting the size in the first element. + if len(ids) == 0 { + p.count = uint16(len(ids)) + } else if len(ids) < 0xFFFF { + p.count = uint16(len(ids)) + copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:], ids) + } else { + p.count = 0xFFFF + ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0] = pgid(len(ids)) + copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[1:], ids) + } + + return nil +} + +// reload reads the freelist from a page and filters out pending items. +func (f *freelist) reload(p *page) { + f.read(p) + + // Build a cache of only pending pages. + pcache := make(map[pgid]bool) + for _, pendingIDs := range f.pending { + for _, pendingID := range pendingIDs { + pcache[pendingID] = true + } + } + + // Check each page in the freelist and build a new available freelist + // with any pages not in the pending lists. + var a []pgid + for _, id := range f.ids { + if !pcache[id] { + a = append(a, id) + } + } + f.ids = a + + // Once the available list is rebuilt then rebuild the free cache so that + // it includes the available and pending free pages. + f.reindex() +} + +// reindex rebuilds the free cache based on available and pending free lists. +func (f *freelist) reindex() { + f.cache = make(map[pgid]bool) + for _, id := range f.ids { + f.cache[id] = true + } + for _, pendingIDs := range f.pending { + for _, pendingID := range pendingIDs { + f.cache[pendingID] = true + } + } +} diff --git a/vendor/github.com/coreos/bbolt/freelist_test.go b/vendor/github.com/coreos/bbolt/freelist_test.go new file mode 100644 index 0000000..4e9b3a8 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/freelist_test.go @@ -0,0 +1,158 @@ +package bolt + +import ( + "math/rand" + "reflect" + "sort" + "testing" + "unsafe" +) + +// Ensure that a page is added to a transaction's freelist. +func TestFreelist_free(t *testing.T) { + f := newFreelist() + f.free(100, &page{id: 12}) + if !reflect.DeepEqual([]pgid{12}, f.pending[100]) { + t.Fatalf("exp=%v; got=%v", []pgid{12}, f.pending[100]) + } +} + +// Ensure that a page and its overflow is added to a transaction's freelist. +func TestFreelist_free_overflow(t *testing.T) { + f := newFreelist() + f.free(100, &page{id: 12, overflow: 3}) + if exp := []pgid{12, 13, 14, 15}; !reflect.DeepEqual(exp, f.pending[100]) { + t.Fatalf("exp=%v; got=%v", exp, f.pending[100]) + } +} + +// Ensure that a transaction's free pages can be released. +func TestFreelist_release(t *testing.T) { + f := newFreelist() + f.free(100, &page{id: 12, overflow: 1}) + f.free(100, &page{id: 9}) + f.free(102, &page{id: 39}) + f.release(100) + f.release(101) + if exp := []pgid{9, 12, 13}; !reflect.DeepEqual(exp, f.ids) { + t.Fatalf("exp=%v; got=%v", exp, f.ids) + } + + f.release(102) + if exp := []pgid{9, 12, 13, 39}; !reflect.DeepEqual(exp, f.ids) { + t.Fatalf("exp=%v; got=%v", exp, f.ids) + } +} + +// Ensure that a freelist can find contiguous blocks of pages. +func TestFreelist_allocate(t *testing.T) { + f := &freelist{ids: []pgid{3, 4, 5, 6, 7, 9, 12, 13, 18}} + if id := int(f.allocate(3)); id != 3 { + t.Fatalf("exp=3; got=%v", id) + } + if id := int(f.allocate(1)); id != 6 { + t.Fatalf("exp=6; got=%v", id) + } + if id := int(f.allocate(3)); id != 0 { + t.Fatalf("exp=0; got=%v", id) + } + if id := int(f.allocate(2)); id != 12 { + t.Fatalf("exp=12; got=%v", id) + } + if id := int(f.allocate(1)); id != 7 { + t.Fatalf("exp=7; got=%v", id) + } + if id := int(f.allocate(0)); id != 0 { + t.Fatalf("exp=0; got=%v", id) + } + if id := int(f.allocate(0)); id != 0 { + t.Fatalf("exp=0; got=%v", id) + } + if exp := []pgid{9, 18}; !reflect.DeepEqual(exp, f.ids) { + t.Fatalf("exp=%v; got=%v", exp, f.ids) + } + + if id := int(f.allocate(1)); id != 9 { + t.Fatalf("exp=9; got=%v", id) + } + if id := int(f.allocate(1)); id != 18 { + t.Fatalf("exp=18; got=%v", id) + } + if id := int(f.allocate(1)); id != 0 { + t.Fatalf("exp=0; got=%v", id) + } + if exp := []pgid{}; !reflect.DeepEqual(exp, f.ids) { + t.Fatalf("exp=%v; got=%v", exp, f.ids) + } +} + +// Ensure that a freelist can deserialize from a freelist page. +func TestFreelist_read(t *testing.T) { + // Create a page. + var buf [4096]byte + page := (*page)(unsafe.Pointer(&buf[0])) + page.flags = freelistPageFlag + page.count = 2 + + // Insert 2 page ids. + ids := (*[3]pgid)(unsafe.Pointer(&page.ptr)) + ids[0] = 23 + ids[1] = 50 + + // Deserialize page into a freelist. + f := newFreelist() + f.read(page) + + // Ensure that there are two page ids in the freelist. + if exp := []pgid{23, 50}; !reflect.DeepEqual(exp, f.ids) { + t.Fatalf("exp=%v; got=%v", exp, f.ids) + } +} + +// Ensure that a freelist can serialize into a freelist page. +func TestFreelist_write(t *testing.T) { + // Create a freelist and write it to a page. + var buf [4096]byte + f := &freelist{ids: []pgid{12, 39}, pending: make(map[txid][]pgid)} + f.pending[100] = []pgid{28, 11} + f.pending[101] = []pgid{3} + p := (*page)(unsafe.Pointer(&buf[0])) + if err := f.write(p); err != nil { + t.Fatal(err) + } + + // Read the page back out. + f2 := newFreelist() + f2.read(p) + + // Ensure that the freelist is correct. + // All pages should be present and in reverse order. + if exp := []pgid{3, 11, 12, 28, 39}; !reflect.DeepEqual(exp, f2.ids) { + t.Fatalf("exp=%v; got=%v", exp, f2.ids) + } +} + +func Benchmark_FreelistRelease10K(b *testing.B) { benchmark_FreelistRelease(b, 10000) } +func Benchmark_FreelistRelease100K(b *testing.B) { benchmark_FreelistRelease(b, 100000) } +func Benchmark_FreelistRelease1000K(b *testing.B) { benchmark_FreelistRelease(b, 1000000) } +func Benchmark_FreelistRelease10000K(b *testing.B) { benchmark_FreelistRelease(b, 10000000) } + +func benchmark_FreelistRelease(b *testing.B, size int) { + ids := randomPgids(size) + pending := randomPgids(len(ids) / 400) + b.ResetTimer() + for i := 0; i < b.N; i++ { + f := &freelist{ids: ids, pending: map[txid][]pgid{1: pending}} + f.release(1) + } +} + +func randomPgids(n int) []pgid { + rand.Seed(42) + pgids := make(pgids, n) + for i := range pgids { + pgids[i] = pgid(rand.Int63()) + } + sort.Sort(pgids) + return pgids +} diff --git a/vendor/github.com/coreos/bbolt/node.go b/vendor/github.com/coreos/bbolt/node.go new file mode 100644 index 0000000..159318b --- /dev/null +++ b/vendor/github.com/coreos/bbolt/node.go @@ -0,0 +1,604 @@ +package bolt + +import ( + "bytes" + "fmt" + "sort" + "unsafe" +) + +// node represents an in-memory, deserialized page. +type node struct { + bucket *Bucket + isLeaf bool + unbalanced bool + spilled bool + key []byte + pgid pgid + parent *node + children nodes + inodes inodes +} + +// root returns the top-level node this node is attached to. +func (n *node) root() *node { + if n.parent == nil { + return n + } + return n.parent.root() +} + +// minKeys returns the minimum number of inodes this node should have. +func (n *node) minKeys() int { + if n.isLeaf { + return 1 + } + return 2 +} + +// size returns the size of the node after serialization. +func (n *node) size() int { + sz, elsz := pageHeaderSize, n.pageElementSize() + for i := 0; i < len(n.inodes); i++ { + item := &n.inodes[i] + sz += elsz + len(item.key) + len(item.value) + } + return sz +} + +// sizeLessThan returns true if the node is less than a given size. +// This is an optimization to avoid calculating a large node when we only need +// to know if it fits inside a certain page size. +func (n *node) sizeLessThan(v int) bool { + sz, elsz := pageHeaderSize, n.pageElementSize() + for i := 0; i < len(n.inodes); i++ { + item := &n.inodes[i] + sz += elsz + len(item.key) + len(item.value) + if sz >= v { + return false + } + } + return true +} + +// pageElementSize returns the size of each page element based on the type of node. +func (n *node) pageElementSize() int { + if n.isLeaf { + return leafPageElementSize + } + return branchPageElementSize +} + +// childAt returns the child node at a given index. +func (n *node) childAt(index int) *node { + if n.isLeaf { + panic(fmt.Sprintf("invalid childAt(%d) on a leaf node", index)) + } + return n.bucket.node(n.inodes[index].pgid, n) +} + +// childIndex returns the index of a given child node. +func (n *node) childIndex(child *node) int { + index := sort.Search(len(n.inodes), func(i int) bool { return bytes.Compare(n.inodes[i].key, child.key) != -1 }) + return index +} + +// numChildren returns the number of children. +func (n *node) numChildren() int { + return len(n.inodes) +} + +// nextSibling returns the next node with the same parent. +func (n *node) nextSibling() *node { + if n.parent == nil { + return nil + } + index := n.parent.childIndex(n) + if index >= n.parent.numChildren()-1 { + return nil + } + return n.parent.childAt(index + 1) +} + +// prevSibling returns the previous node with the same parent. +func (n *node) prevSibling() *node { + if n.parent == nil { + return nil + } + index := n.parent.childIndex(n) + if index == 0 { + return nil + } + return n.parent.childAt(index - 1) +} + +// put inserts a key/value. +func (n *node) put(oldKey, newKey, value []byte, pgid pgid, flags uint32) { + if pgid >= n.bucket.tx.meta.pgid { + panic(fmt.Sprintf("pgid (%d) above high water mark (%d)", pgid, n.bucket.tx.meta.pgid)) + } else if len(oldKey) <= 0 { + panic("put: zero-length old key") + } else if len(newKey) <= 0 { + panic("put: zero-length new key") + } + + // Find insertion index. + index := sort.Search(len(n.inodes), func(i int) bool { return bytes.Compare(n.inodes[i].key, oldKey) != -1 }) + + // Add capacity and shift nodes if we don't have an exact match and need to insert. + exact := (len(n.inodes) > 0 && index < len(n.inodes) && bytes.Equal(n.inodes[index].key, oldKey)) + if !exact { + n.inodes = append(n.inodes, inode{}) + copy(n.inodes[index+1:], n.inodes[index:]) + } + + inode := &n.inodes[index] + inode.flags = flags + inode.key = newKey + inode.value = value + inode.pgid = pgid + _assert(len(inode.key) > 0, "put: zero-length inode key") +} + +// del removes a key from the node. +func (n *node) del(key []byte) { + // Find index of key. + index := sort.Search(len(n.inodes), func(i int) bool { return bytes.Compare(n.inodes[i].key, key) != -1 }) + + // Exit if the key isn't found. + if index >= len(n.inodes) || !bytes.Equal(n.inodes[index].key, key) { + return + } + + // Delete inode from the node. + n.inodes = append(n.inodes[:index], n.inodes[index+1:]...) + + // Mark the node as needing rebalancing. + n.unbalanced = true +} + +// read initializes the node from a page. +func (n *node) read(p *page) { + n.pgid = p.id + n.isLeaf = ((p.flags & leafPageFlag) != 0) + n.inodes = make(inodes, int(p.count)) + + for i := 0; i < int(p.count); i++ { + inode := &n.inodes[i] + if n.isLeaf { + elem := p.leafPageElement(uint16(i)) + inode.flags = elem.flags + inode.key = elem.key() + inode.value = elem.value() + } else { + elem := p.branchPageElement(uint16(i)) + inode.pgid = elem.pgid + inode.key = elem.key() + } + _assert(len(inode.key) > 0, "read: zero-length inode key") + } + + // Save first key so we can find the node in the parent when we spill. + if len(n.inodes) > 0 { + n.key = n.inodes[0].key + _assert(len(n.key) > 0, "read: zero-length node key") + } else { + n.key = nil + } +} + +// write writes the items onto one or more pages. +func (n *node) write(p *page) { + // Initialize page. + if n.isLeaf { + p.flags |= leafPageFlag + } else { + p.flags |= branchPageFlag + } + + if len(n.inodes) >= 0xFFFF { + panic(fmt.Sprintf("inode overflow: %d (pgid=%d)", len(n.inodes), p.id)) + } + p.count = uint16(len(n.inodes)) + + // Stop here if there are no items to write. + if p.count == 0 { + return + } + + // Loop over each item and write it to the page. + b := (*[maxAllocSize]byte)(unsafe.Pointer(&p.ptr))[n.pageElementSize()*len(n.inodes):] + for i, item := range n.inodes { + _assert(len(item.key) > 0, "write: zero-length inode key") + + // Write the page element. + if n.isLeaf { + elem := p.leafPageElement(uint16(i)) + elem.pos = uint32(uintptr(unsafe.Pointer(&b[0])) - uintptr(unsafe.Pointer(elem))) + elem.flags = item.flags + elem.ksize = uint32(len(item.key)) + elem.vsize = uint32(len(item.value)) + } else { + elem := p.branchPageElement(uint16(i)) + elem.pos = uint32(uintptr(unsafe.Pointer(&b[0])) - uintptr(unsafe.Pointer(elem))) + elem.ksize = uint32(len(item.key)) + elem.pgid = item.pgid + _assert(elem.pgid != p.id, "write: circular dependency occurred") + } + + // If the length of key+value is larger than the max allocation size + // then we need to reallocate the byte array pointer. + // + // See: https://github.com/boltdb/bolt/pull/335 + klen, vlen := len(item.key), len(item.value) + if len(b) < klen+vlen { + b = (*[maxAllocSize]byte)(unsafe.Pointer(&b[0]))[:] + } + + // Write data for the element to the end of the page. + copy(b[0:], item.key) + b = b[klen:] + copy(b[0:], item.value) + b = b[vlen:] + } + + // DEBUG ONLY: n.dump() +} + +// split breaks up a node into multiple smaller nodes, if appropriate. +// This should only be called from the spill() function. +func (n *node) split(pageSize int) []*node { + var nodes []*node + + node := n + for { + // Split node into two. + a, b := node.splitTwo(pageSize) + nodes = append(nodes, a) + + // If we can't split then exit the loop. + if b == nil { + break + } + + // Set node to b so it gets split on the next iteration. + node = b + } + + return nodes +} + +// splitTwo breaks up a node into two smaller nodes, if appropriate. +// This should only be called from the split() function. +func (n *node) splitTwo(pageSize int) (*node, *node) { + // Ignore the split if the page doesn't have at least enough nodes for + // two pages or if the nodes can fit in a single page. + if len(n.inodes) <= (minKeysPerPage*2) || n.sizeLessThan(pageSize) { + return n, nil + } + + // Determine the threshold before starting a new node. + var fillPercent = n.bucket.FillPercent + if fillPercent < minFillPercent { + fillPercent = minFillPercent + } else if fillPercent > maxFillPercent { + fillPercent = maxFillPercent + } + threshold := int(float64(pageSize) * fillPercent) + + // Determine split position and sizes of the two pages. + splitIndex, _ := n.splitIndex(threshold) + + // Split node into two separate nodes. + // If there's no parent then we'll need to create one. + if n.parent == nil { + n.parent = &node{bucket: n.bucket, children: []*node{n}} + } + + // Create a new node and add it to the parent. + next := &node{bucket: n.bucket, isLeaf: n.isLeaf, parent: n.parent} + n.parent.children = append(n.parent.children, next) + + // Split inodes across two nodes. + next.inodes = n.inodes[splitIndex:] + n.inodes = n.inodes[:splitIndex] + + // Update the statistics. + n.bucket.tx.stats.Split++ + + return n, next +} + +// splitIndex finds the position where a page will fill a given threshold. +// It returns the index as well as the size of the first page. +// This is only be called from split(). +func (n *node) splitIndex(threshold int) (index, sz int) { + sz = pageHeaderSize + + // Loop until we only have the minimum number of keys required for the second page. + for i := 0; i < len(n.inodes)-minKeysPerPage; i++ { + index = i + inode := n.inodes[i] + elsize := n.pageElementSize() + len(inode.key) + len(inode.value) + + // If we have at least the minimum number of keys and adding another + // node would put us over the threshold then exit and return. + if i >= minKeysPerPage && sz+elsize > threshold { + break + } + + // Add the element size to the total size. + sz += elsize + } + + return +} + +// spill writes the nodes to dirty pages and splits nodes as it goes. +// Returns an error if dirty pages cannot be allocated. +func (n *node) spill() error { + var tx = n.bucket.tx + if n.spilled { + return nil + } + + // Spill child nodes first. Child nodes can materialize sibling nodes in + // the case of split-merge so we cannot use a range loop. We have to check + // the children size on every loop iteration. + sort.Sort(n.children) + for i := 0; i < len(n.children); i++ { + if err := n.children[i].spill(); err != nil { + return err + } + } + + // We no longer need the child list because it's only used for spill tracking. + n.children = nil + + // Split nodes into appropriate sizes. The first node will always be n. + var nodes = n.split(tx.db.pageSize) + for _, node := range nodes { + // Add node's page to the freelist if it's not new. + if node.pgid > 0 { + tx.db.freelist.free(tx.meta.txid, tx.page(node.pgid)) + node.pgid = 0 + } + + // Allocate contiguous space for the node. + p, err := tx.allocate((node.size() / tx.db.pageSize) + 1) + if err != nil { + return err + } + + // Write the node. + if p.id >= tx.meta.pgid { + panic(fmt.Sprintf("pgid (%d) above high water mark (%d)", p.id, tx.meta.pgid)) + } + node.pgid = p.id + node.write(p) + node.spilled = true + + // Insert into parent inodes. + if node.parent != nil { + var key = node.key + if key == nil { + key = node.inodes[0].key + } + + node.parent.put(key, node.inodes[0].key, nil, node.pgid, 0) + node.key = node.inodes[0].key + _assert(len(node.key) > 0, "spill: zero-length node key") + } + + // Update the statistics. + tx.stats.Spill++ + } + + // If the root node split and created a new root then we need to spill that + // as well. We'll clear out the children to make sure it doesn't try to respill. + if n.parent != nil && n.parent.pgid == 0 { + n.children = nil + return n.parent.spill() + } + + return nil +} + +// rebalance attempts to combine the node with sibling nodes if the node fill +// size is below a threshold or if there are not enough keys. +func (n *node) rebalance() { + if !n.unbalanced { + return + } + n.unbalanced = false + + // Update statistics. + n.bucket.tx.stats.Rebalance++ + + // Ignore if node is above threshold (25%) and has enough keys. + var threshold = n.bucket.tx.db.pageSize / 4 + if n.size() > threshold && len(n.inodes) > n.minKeys() { + return + } + + // Root node has special handling. + if n.parent == nil { + // If root node is a branch and only has one node then collapse it. + if !n.isLeaf && len(n.inodes) == 1 { + // Move root's child up. + child := n.bucket.node(n.inodes[0].pgid, n) + n.isLeaf = child.isLeaf + n.inodes = child.inodes[:] + n.children = child.children + + // Reparent all child nodes being moved. + for _, inode := range n.inodes { + if child, ok := n.bucket.nodes[inode.pgid]; ok { + child.parent = n + } + } + + // Remove old child. + child.parent = nil + delete(n.bucket.nodes, child.pgid) + child.free() + } + + return + } + + // If node has no keys then just remove it. + if n.numChildren() == 0 { + n.parent.del(n.key) + n.parent.removeChild(n) + delete(n.bucket.nodes, n.pgid) + n.free() + n.parent.rebalance() + return + } + + _assert(n.parent.numChildren() > 1, "parent must have at least 2 children") + + // Destination node is right sibling if idx == 0, otherwise left sibling. + var target *node + var useNextSibling = (n.parent.childIndex(n) == 0) + if useNextSibling { + target = n.nextSibling() + } else { + target = n.prevSibling() + } + + // If both this node and the target node are too small then merge them. + if useNextSibling { + // Reparent all child nodes being moved. + for _, inode := range target.inodes { + if child, ok := n.bucket.nodes[inode.pgid]; ok { + child.parent.removeChild(child) + child.parent = n + child.parent.children = append(child.parent.children, child) + } + } + + // Copy over inodes from target and remove target. + n.inodes = append(n.inodes, target.inodes...) + n.parent.del(target.key) + n.parent.removeChild(target) + delete(n.bucket.nodes, target.pgid) + target.free() + } else { + // Reparent all child nodes being moved. + for _, inode := range n.inodes { + if child, ok := n.bucket.nodes[inode.pgid]; ok { + child.parent.removeChild(child) + child.parent = target + child.parent.children = append(child.parent.children, child) + } + } + + // Copy over inodes to target and remove node. + target.inodes = append(target.inodes, n.inodes...) + n.parent.del(n.key) + n.parent.removeChild(n) + delete(n.bucket.nodes, n.pgid) + n.free() + } + + // Either this node or the target node was deleted from the parent so rebalance it. + n.parent.rebalance() +} + +// removes a node from the list of in-memory children. +// This does not affect the inodes. +func (n *node) removeChild(target *node) { + for i, child := range n.children { + if child == target { + n.children = append(n.children[:i], n.children[i+1:]...) + return + } + } +} + +// dereference causes the node to copy all its inode key/value references to heap memory. +// This is required when the mmap is reallocated so inodes are not pointing to stale data. +func (n *node) dereference() { + if n.key != nil { + key := make([]byte, len(n.key)) + copy(key, n.key) + n.key = key + _assert(n.pgid == 0 || len(n.key) > 0, "dereference: zero-length node key on existing node") + } + + for i := range n.inodes { + inode := &n.inodes[i] + + key := make([]byte, len(inode.key)) + copy(key, inode.key) + inode.key = key + _assert(len(inode.key) > 0, "dereference: zero-length inode key") + + value := make([]byte, len(inode.value)) + copy(value, inode.value) + inode.value = value + } + + // Recursively dereference children. + for _, child := range n.children { + child.dereference() + } + + // Update statistics. + n.bucket.tx.stats.NodeDeref++ +} + +// free adds the node's underlying page to the freelist. +func (n *node) free() { + if n.pgid != 0 { + n.bucket.tx.db.freelist.free(n.bucket.tx.meta.txid, n.bucket.tx.page(n.pgid)) + n.pgid = 0 + } +} + +// dump writes the contents of the node to STDERR for debugging purposes. +/* +func (n *node) dump() { + // Write node header. + var typ = "branch" + if n.isLeaf { + typ = "leaf" + } + warnf("[NODE %d {type=%s count=%d}]", n.pgid, typ, len(n.inodes)) + + // Write out abbreviated version of each item. + for _, item := range n.inodes { + if n.isLeaf { + if item.flags&bucketLeafFlag != 0 { + bucket := (*bucket)(unsafe.Pointer(&item.value[0])) + warnf("+L %08x -> (bucket root=%d)", trunc(item.key, 4), bucket.root) + } else { + warnf("+L %08x -> %08x", trunc(item.key, 4), trunc(item.value, 4)) + } + } else { + warnf("+B %08x -> pgid=%d", trunc(item.key, 4), item.pgid) + } + } + warn("") +} +*/ + +type nodes []*node + +func (s nodes) Len() int { return len(s) } +func (s nodes) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s nodes) Less(i, j int) bool { return bytes.Compare(s[i].inodes[0].key, s[j].inodes[0].key) == -1 } + +// inode represents an internal node inside of a node. +// It can be used to point to elements in a page or point +// to an element which hasn't been added to a page yet. +type inode struct { + flags uint32 + pgid pgid + key []byte + value []byte +} + +type inodes []inode diff --git a/vendor/github.com/coreos/bbolt/node_test.go b/vendor/github.com/coreos/bbolt/node_test.go new file mode 100644 index 0000000..fa5d10f --- /dev/null +++ b/vendor/github.com/coreos/bbolt/node_test.go @@ -0,0 +1,156 @@ +package bolt + +import ( + "testing" + "unsafe" +) + +// Ensure that a node can insert a key/value. +func TestNode_put(t *testing.T) { + n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{meta: &meta{pgid: 1}}}} + n.put([]byte("baz"), []byte("baz"), []byte("2"), 0, 0) + n.put([]byte("foo"), []byte("foo"), []byte("0"), 0, 0) + n.put([]byte("bar"), []byte("bar"), []byte("1"), 0, 0) + n.put([]byte("foo"), []byte("foo"), []byte("3"), 0, leafPageFlag) + + if len(n.inodes) != 3 { + t.Fatalf("exp=3; got=%d", len(n.inodes)) + } + if k, v := n.inodes[0].key, n.inodes[0].value; string(k) != "bar" || string(v) != "1" { + t.Fatalf("exp=; got=<%s,%s>", k, v) + } + if k, v := n.inodes[1].key, n.inodes[1].value; string(k) != "baz" || string(v) != "2" { + t.Fatalf("exp=; got=<%s,%s>", k, v) + } + if k, v := n.inodes[2].key, n.inodes[2].value; string(k) != "foo" || string(v) != "3" { + t.Fatalf("exp=; got=<%s,%s>", k, v) + } + if n.inodes[2].flags != uint32(leafPageFlag) { + t.Fatalf("not a leaf: %d", n.inodes[2].flags) + } +} + +// Ensure that a node can deserialize from a leaf page. +func TestNode_read_LeafPage(t *testing.T) { + // Create a page. + var buf [4096]byte + page := (*page)(unsafe.Pointer(&buf[0])) + page.flags = leafPageFlag + page.count = 2 + + // Insert 2 elements at the beginning. sizeof(leafPageElement) == 16 + nodes := (*[3]leafPageElement)(unsafe.Pointer(&page.ptr)) + nodes[0] = leafPageElement{flags: 0, pos: 32, ksize: 3, vsize: 4} // pos = sizeof(leafPageElement) * 2 + nodes[1] = leafPageElement{flags: 0, pos: 23, ksize: 10, vsize: 3} // pos = sizeof(leafPageElement) + 3 + 4 + + // Write data for the nodes at the end. + data := (*[4096]byte)(unsafe.Pointer(&nodes[2])) + copy(data[:], []byte("barfooz")) + copy(data[7:], []byte("helloworldbye")) + + // Deserialize page into a leaf. + n := &node{} + n.read(page) + + // Check that there are two inodes with correct data. + if !n.isLeaf { + t.Fatal("expected leaf") + } + if len(n.inodes) != 2 { + t.Fatalf("exp=2; got=%d", len(n.inodes)) + } + if k, v := n.inodes[0].key, n.inodes[0].value; string(k) != "bar" || string(v) != "fooz" { + t.Fatalf("exp=; got=<%s,%s>", k, v) + } + if k, v := n.inodes[1].key, n.inodes[1].value; string(k) != "helloworld" || string(v) != "bye" { + t.Fatalf("exp=; got=<%s,%s>", k, v) + } +} + +// Ensure that a node can serialize into a leaf page. +func TestNode_write_LeafPage(t *testing.T) { + // Create a node. + n := &node{isLeaf: true, inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{db: &DB{}, meta: &meta{pgid: 1}}}} + n.put([]byte("susy"), []byte("susy"), []byte("que"), 0, 0) + n.put([]byte("ricki"), []byte("ricki"), []byte("lake"), 0, 0) + n.put([]byte("john"), []byte("john"), []byte("johnson"), 0, 0) + + // Write it to a page. + var buf [4096]byte + p := (*page)(unsafe.Pointer(&buf[0])) + n.write(p) + + // Read the page back in. + n2 := &node{} + n2.read(p) + + // Check that the two pages are the same. + if len(n2.inodes) != 3 { + t.Fatalf("exp=3; got=%d", len(n2.inodes)) + } + if k, v := n2.inodes[0].key, n2.inodes[0].value; string(k) != "john" || string(v) != "johnson" { + t.Fatalf("exp=; got=<%s,%s>", k, v) + } + if k, v := n2.inodes[1].key, n2.inodes[1].value; string(k) != "ricki" || string(v) != "lake" { + t.Fatalf("exp=; got=<%s,%s>", k, v) + } + if k, v := n2.inodes[2].key, n2.inodes[2].value; string(k) != "susy" || string(v) != "que" { + t.Fatalf("exp=; got=<%s,%s>", k, v) + } +} + +// Ensure that a node can split into appropriate subgroups. +func TestNode_split(t *testing.T) { + // Create a node. + n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{db: &DB{}, meta: &meta{pgid: 1}}}} + n.put([]byte("00000001"), []byte("00000001"), []byte("0123456701234567"), 0, 0) + n.put([]byte("00000002"), []byte("00000002"), []byte("0123456701234567"), 0, 0) + n.put([]byte("00000003"), []byte("00000003"), []byte("0123456701234567"), 0, 0) + n.put([]byte("00000004"), []byte("00000004"), []byte("0123456701234567"), 0, 0) + n.put([]byte("00000005"), []byte("00000005"), []byte("0123456701234567"), 0, 0) + + // Split between 2 & 3. + n.split(100) + + var parent = n.parent + if len(parent.children) != 2 { + t.Fatalf("exp=2; got=%d", len(parent.children)) + } + if len(parent.children[0].inodes) != 2 { + t.Fatalf("exp=2; got=%d", len(parent.children[0].inodes)) + } + if len(parent.children[1].inodes) != 3 { + t.Fatalf("exp=3; got=%d", len(parent.children[1].inodes)) + } +} + +// Ensure that a page with the minimum number of inodes just returns a single node. +func TestNode_split_MinKeys(t *testing.T) { + // Create a node. + n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{db: &DB{}, meta: &meta{pgid: 1}}}} + n.put([]byte("00000001"), []byte("00000001"), []byte("0123456701234567"), 0, 0) + n.put([]byte("00000002"), []byte("00000002"), []byte("0123456701234567"), 0, 0) + + // Split. + n.split(20) + if n.parent != nil { + t.Fatalf("expected nil parent") + } +} + +// Ensure that a node that has keys that all fit on a page just returns one leaf. +func TestNode_split_SinglePage(t *testing.T) { + // Create a node. + n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{db: &DB{}, meta: &meta{pgid: 1}}}} + n.put([]byte("00000001"), []byte("00000001"), []byte("0123456701234567"), 0, 0) + n.put([]byte("00000002"), []byte("00000002"), []byte("0123456701234567"), 0, 0) + n.put([]byte("00000003"), []byte("00000003"), []byte("0123456701234567"), 0, 0) + n.put([]byte("00000004"), []byte("00000004"), []byte("0123456701234567"), 0, 0) + n.put([]byte("00000005"), []byte("00000005"), []byte("0123456701234567"), 0, 0) + + // Split. + n.split(4096) + if n.parent != nil { + t.Fatalf("expected nil parent") + } +} diff --git a/vendor/github.com/coreos/bbolt/page.go b/vendor/github.com/coreos/bbolt/page.go new file mode 100644 index 0000000..7651a6b --- /dev/null +++ b/vendor/github.com/coreos/bbolt/page.go @@ -0,0 +1,178 @@ +package bolt + +import ( + "fmt" + "os" + "sort" + "unsafe" +) + +const pageHeaderSize = int(unsafe.Offsetof(((*page)(nil)).ptr)) + +const minKeysPerPage = 2 + +const branchPageElementSize = int(unsafe.Sizeof(branchPageElement{})) +const leafPageElementSize = int(unsafe.Sizeof(leafPageElement{})) + +const ( + branchPageFlag = 0x01 + leafPageFlag = 0x02 + metaPageFlag = 0x04 + freelistPageFlag = 0x10 +) + +const ( + bucketLeafFlag = 0x01 +) + +type pgid uint64 + +type page struct { + id pgid + flags uint16 + count uint16 + overflow uint32 + ptr uintptr +} + +// typ returns a human readable page type string used for debugging. +func (p *page) typ() string { + if (p.flags & branchPageFlag) != 0 { + return "branch" + } else if (p.flags & leafPageFlag) != 0 { + return "leaf" + } else if (p.flags & metaPageFlag) != 0 { + return "meta" + } else if (p.flags & freelistPageFlag) != 0 { + return "freelist" + } + return fmt.Sprintf("unknown<%02x>", p.flags) +} + +// meta returns a pointer to the metadata section of the page. +func (p *page) meta() *meta { + return (*meta)(unsafe.Pointer(&p.ptr)) +} + +// leafPageElement retrieves the leaf node by index +func (p *page) leafPageElement(index uint16) *leafPageElement { + n := &((*[0x7FFFFFF]leafPageElement)(unsafe.Pointer(&p.ptr)))[index] + return n +} + +// leafPageElements retrieves a list of leaf nodes. +func (p *page) leafPageElements() []leafPageElement { + if p.count == 0 { + return nil + } + return ((*[0x7FFFFFF]leafPageElement)(unsafe.Pointer(&p.ptr)))[:] +} + +// branchPageElement retrieves the branch node by index +func (p *page) branchPageElement(index uint16) *branchPageElement { + return &((*[0x7FFFFFF]branchPageElement)(unsafe.Pointer(&p.ptr)))[index] +} + +// branchPageElements retrieves a list of branch nodes. +func (p *page) branchPageElements() []branchPageElement { + if p.count == 0 { + return nil + } + return ((*[0x7FFFFFF]branchPageElement)(unsafe.Pointer(&p.ptr)))[:] +} + +// dump writes n bytes of the page to STDERR as hex output. +func (p *page) hexdump(n int) { + buf := (*[maxAllocSize]byte)(unsafe.Pointer(p))[:n] + fmt.Fprintf(os.Stderr, "%x\n", buf) +} + +type pages []*page + +func (s pages) Len() int { return len(s) } +func (s pages) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s pages) Less(i, j int) bool { return s[i].id < s[j].id } + +// branchPageElement represents a node on a branch page. +type branchPageElement struct { + pos uint32 + ksize uint32 + pgid pgid +} + +// key returns a byte slice of the node key. +func (n *branchPageElement) key() []byte { + buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) + return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize] +} + +// leafPageElement represents a node on a leaf page. +type leafPageElement struct { + flags uint32 + pos uint32 + ksize uint32 + vsize uint32 +} + +// key returns a byte slice of the node key. +func (n *leafPageElement) key() []byte { + buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) + return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize:n.ksize] +} + +// value returns a byte slice of the node value. +func (n *leafPageElement) value() []byte { + buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) + return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos+n.ksize]))[:n.vsize:n.vsize] +} + +// PageInfo represents human readable information about a page. +type PageInfo struct { + ID int + Type string + Count int + OverflowCount int +} + +type pgids []pgid + +func (s pgids) Len() int { return len(s) } +func (s pgids) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s pgids) Less(i, j int) bool { return s[i] < s[j] } + +// merge returns the sorted union of a and b. +func (a pgids) merge(b pgids) pgids { + // Return the opposite slice if one is nil. + if len(a) == 0 { + return b + } else if len(b) == 0 { + return a + } + + // Create a list to hold all elements from both lists. + merged := make(pgids, 0, len(a)+len(b)) + + // Assign lead to the slice with a lower starting value, follow to the higher value. + lead, follow := a, b + if b[0] < a[0] { + lead, follow = b, a + } + + // Continue while there are elements in the lead. + for len(lead) > 0 { + // Merge largest prefix of lead that is ahead of follow[0]. + n := sort.Search(len(lead), func(i int) bool { return lead[i] > follow[0] }) + merged = append(merged, lead[:n]...) + if n >= len(lead) { + break + } + + // Swap lead and follow. + lead, follow = follow, lead[n:] + } + + // Append what's left in follow. + merged = append(merged, follow...) + + return merged +} diff --git a/vendor/github.com/coreos/bbolt/page_test.go b/vendor/github.com/coreos/bbolt/page_test.go new file mode 100644 index 0000000..59f4a30 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/page_test.go @@ -0,0 +1,72 @@ +package bolt + +import ( + "reflect" + "sort" + "testing" + "testing/quick" +) + +// Ensure that the page type can be returned in human readable format. +func TestPage_typ(t *testing.T) { + if typ := (&page{flags: branchPageFlag}).typ(); typ != "branch" { + t.Fatalf("exp=branch; got=%v", typ) + } + if typ := (&page{flags: leafPageFlag}).typ(); typ != "leaf" { + t.Fatalf("exp=leaf; got=%v", typ) + } + if typ := (&page{flags: metaPageFlag}).typ(); typ != "meta" { + t.Fatalf("exp=meta; got=%v", typ) + } + if typ := (&page{flags: freelistPageFlag}).typ(); typ != "freelist" { + t.Fatalf("exp=freelist; got=%v", typ) + } + if typ := (&page{flags: 20000}).typ(); typ != "unknown<4e20>" { + t.Fatalf("exp=unknown<4e20>; got=%v", typ) + } +} + +// Ensure that the hexdump debugging function doesn't blow up. +func TestPage_dump(t *testing.T) { + (&page{id: 256}).hexdump(16) +} + +func TestPgids_merge(t *testing.T) { + a := pgids{4, 5, 6, 10, 11, 12, 13, 27} + b := pgids{1, 3, 8, 9, 25, 30} + c := a.merge(b) + if !reflect.DeepEqual(c, pgids{1, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 25, 27, 30}) { + t.Errorf("mismatch: %v", c) + } + + a = pgids{4, 5, 6, 10, 11, 12, 13, 27, 35, 36} + b = pgids{8, 9, 25, 30} + c = a.merge(b) + if !reflect.DeepEqual(c, pgids{4, 5, 6, 8, 9, 10, 11, 12, 13, 25, 27, 30, 35, 36}) { + t.Errorf("mismatch: %v", c) + } +} + +func TestPgids_merge_quick(t *testing.T) { + if err := quick.Check(func(a, b pgids) bool { + // Sort incoming lists. + sort.Sort(a) + sort.Sort(b) + + // Merge the two lists together. + got := a.merge(b) + + // The expected value should be the two lists combined and sorted. + exp := append(a, b...) + sort.Sort(exp) + + if !reflect.DeepEqual(exp, got) { + t.Errorf("\nexp=%+v\ngot=%+v\n", exp, got) + return false + } + + return true + }, nil); err != nil { + t.Fatal(err) + } +} diff --git a/vendor/github.com/coreos/bbolt/quick_test.go b/vendor/github.com/coreos/bbolt/quick_test.go new file mode 100644 index 0000000..4da5817 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/quick_test.go @@ -0,0 +1,79 @@ +package bolt_test + +import ( + "bytes" + "flag" + "fmt" + "math/rand" + "os" + "reflect" + "testing/quick" + "time" +) + +// testing/quick defaults to 5 iterations and a random seed. +// You can override these settings from the command line: +// +// -quick.count The number of iterations to perform. +// -quick.seed The seed to use for randomizing. +// -quick.maxitems The maximum number of items to insert into a DB. +// -quick.maxksize The maximum size of a key. +// -quick.maxvsize The maximum size of a value. +// + +var qcount, qseed, qmaxitems, qmaxksize, qmaxvsize int + +func init() { + flag.IntVar(&qcount, "quick.count", 5, "") + flag.IntVar(&qseed, "quick.seed", int(time.Now().UnixNano())%100000, "") + flag.IntVar(&qmaxitems, "quick.maxitems", 1000, "") + flag.IntVar(&qmaxksize, "quick.maxksize", 1024, "") + flag.IntVar(&qmaxvsize, "quick.maxvsize", 1024, "") + flag.Parse() + fmt.Fprintln(os.Stderr, "seed:", qseed) + fmt.Fprintf(os.Stderr, "quick settings: count=%v, items=%v, ksize=%v, vsize=%v\n", qcount, qmaxitems, qmaxksize, qmaxvsize) +} + +func qconfig() *quick.Config { + return &quick.Config{ + MaxCount: qcount, + Rand: rand.New(rand.NewSource(int64(qseed))), + } +} + +type testdata []testdataitem + +func (t testdata) Len() int { return len(t) } +func (t testdata) Swap(i, j int) { t[i], t[j] = t[j], t[i] } +func (t testdata) Less(i, j int) bool { return bytes.Compare(t[i].Key, t[j].Key) == -1 } + +func (t testdata) Generate(rand *rand.Rand, size int) reflect.Value { + n := rand.Intn(qmaxitems-1) + 1 + items := make(testdata, n) + for i := 0; i < n; i++ { + item := &items[i] + item.Key = randByteSlice(rand, 1, qmaxksize) + item.Value = randByteSlice(rand, 0, qmaxvsize) + } + return reflect.ValueOf(items) +} + +type revtestdata []testdataitem + +func (t revtestdata) Len() int { return len(t) } +func (t revtestdata) Swap(i, j int) { t[i], t[j] = t[j], t[i] } +func (t revtestdata) Less(i, j int) bool { return bytes.Compare(t[i].Key, t[j].Key) == 1 } + +type testdataitem struct { + Key []byte + Value []byte +} + +func randByteSlice(rand *rand.Rand, minSize, maxSize int) []byte { + n := rand.Intn(maxSize-minSize) + minSize + b := make([]byte, n) + for i := 0; i < n; i++ { + b[i] = byte(rand.Intn(255)) + } + return b +} diff --git a/vendor/github.com/coreos/bbolt/simulation_test.go b/vendor/github.com/coreos/bbolt/simulation_test.go new file mode 100644 index 0000000..3831016 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/simulation_test.go @@ -0,0 +1,329 @@ +package bolt_test + +import ( + "bytes" + "fmt" + "math/rand" + "sync" + "testing" + + "github.com/boltdb/bolt" +) + +func TestSimulate_1op_1p(t *testing.T) { testSimulate(t, 1, 1) } +func TestSimulate_10op_1p(t *testing.T) { testSimulate(t, 10, 1) } +func TestSimulate_100op_1p(t *testing.T) { testSimulate(t, 100, 1) } +func TestSimulate_1000op_1p(t *testing.T) { testSimulate(t, 1000, 1) } +func TestSimulate_10000op_1p(t *testing.T) { testSimulate(t, 10000, 1) } + +func TestSimulate_10op_10p(t *testing.T) { testSimulate(t, 10, 10) } +func TestSimulate_100op_10p(t *testing.T) { testSimulate(t, 100, 10) } +func TestSimulate_1000op_10p(t *testing.T) { testSimulate(t, 1000, 10) } +func TestSimulate_10000op_10p(t *testing.T) { testSimulate(t, 10000, 10) } + +func TestSimulate_100op_100p(t *testing.T) { testSimulate(t, 100, 100) } +func TestSimulate_1000op_100p(t *testing.T) { testSimulate(t, 1000, 100) } +func TestSimulate_10000op_100p(t *testing.T) { testSimulate(t, 10000, 100) } + +func TestSimulate_10000op_1000p(t *testing.T) { testSimulate(t, 10000, 1000) } + +// Randomly generate operations on a given database with multiple clients to ensure consistency and thread safety. +func testSimulate(t *testing.T, threadCount, parallelism int) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + rand.Seed(int64(qseed)) + + // A list of operations that readers and writers can perform. + var readerHandlers = []simulateHandler{simulateGetHandler} + var writerHandlers = []simulateHandler{simulateGetHandler, simulatePutHandler} + + var versions = make(map[int]*QuickDB) + versions[1] = NewQuickDB() + + db := MustOpenDB() + defer db.MustClose() + + var mutex sync.Mutex + + // Run n threads in parallel, each with their own operation. + var wg sync.WaitGroup + var threads = make(chan bool, parallelism) + var i int + for { + threads <- true + wg.Add(1) + writable := ((rand.Int() % 100) < 20) // 20% writers + + // Choose an operation to execute. + var handler simulateHandler + if writable { + handler = writerHandlers[rand.Intn(len(writerHandlers))] + } else { + handler = readerHandlers[rand.Intn(len(readerHandlers))] + } + + // Execute a thread for the given operation. + go func(writable bool, handler simulateHandler) { + defer wg.Done() + + // Start transaction. + tx, err := db.Begin(writable) + if err != nil { + t.Fatal("tx begin: ", err) + } + + // Obtain current state of the dataset. + mutex.Lock() + var qdb = versions[tx.ID()] + if writable { + qdb = versions[tx.ID()-1].Copy() + } + mutex.Unlock() + + // Make sure we commit/rollback the tx at the end and update the state. + if writable { + defer func() { + mutex.Lock() + versions[tx.ID()] = qdb + mutex.Unlock() + + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + }() + } else { + defer func() { _ = tx.Rollback() }() + } + + // Ignore operation if we don't have data yet. + if qdb == nil { + return + } + + // Execute handler. + handler(tx, qdb) + + // Release a thread back to the scheduling loop. + <-threads + }(writable, handler) + + i++ + if i > threadCount { + break + } + } + + // Wait until all threads are done. + wg.Wait() +} + +type simulateHandler func(tx *bolt.Tx, qdb *QuickDB) + +// Retrieves a key from the database and verifies that it is what is expected. +func simulateGetHandler(tx *bolt.Tx, qdb *QuickDB) { + // Randomly retrieve an existing exist. + keys := qdb.Rand() + if len(keys) == 0 { + return + } + + // Retrieve root bucket. + b := tx.Bucket(keys[0]) + if b == nil { + panic(fmt.Sprintf("bucket[0] expected: %08x\n", trunc(keys[0], 4))) + } + + // Drill into nested buckets. + for _, key := range keys[1 : len(keys)-1] { + b = b.Bucket(key) + if b == nil { + panic(fmt.Sprintf("bucket[n] expected: %v -> %v\n", keys, key)) + } + } + + // Verify key/value on the final bucket. + expected := qdb.Get(keys) + actual := b.Get(keys[len(keys)-1]) + if !bytes.Equal(actual, expected) { + fmt.Println("=== EXPECTED ===") + fmt.Println(expected) + fmt.Println("=== ACTUAL ===") + fmt.Println(actual) + fmt.Println("=== END ===") + panic("value mismatch") + } +} + +// Inserts a key into the database. +func simulatePutHandler(tx *bolt.Tx, qdb *QuickDB) { + var err error + keys, value := randKeys(), randValue() + + // Retrieve root bucket. + b := tx.Bucket(keys[0]) + if b == nil { + b, err = tx.CreateBucket(keys[0]) + if err != nil { + panic("create bucket: " + err.Error()) + } + } + + // Create nested buckets, if necessary. + for _, key := range keys[1 : len(keys)-1] { + child := b.Bucket(key) + if child != nil { + b = child + } else { + b, err = b.CreateBucket(key) + if err != nil { + panic("create bucket: " + err.Error()) + } + } + } + + // Insert into database. + if err := b.Put(keys[len(keys)-1], value); err != nil { + panic("put: " + err.Error()) + } + + // Insert into in-memory database. + qdb.Put(keys, value) +} + +// QuickDB is an in-memory database that replicates the functionality of the +// Bolt DB type except that it is entirely in-memory. It is meant for testing +// that the Bolt database is consistent. +type QuickDB struct { + sync.RWMutex + m map[string]interface{} +} + +// NewQuickDB returns an instance of QuickDB. +func NewQuickDB() *QuickDB { + return &QuickDB{m: make(map[string]interface{})} +} + +// Get retrieves the value at a key path. +func (db *QuickDB) Get(keys [][]byte) []byte { + db.RLock() + defer db.RUnlock() + + m := db.m + for _, key := range keys[:len(keys)-1] { + value := m[string(key)] + if value == nil { + return nil + } + switch value := value.(type) { + case map[string]interface{}: + m = value + case []byte: + return nil + } + } + + // Only return if it's a simple value. + if value, ok := m[string(keys[len(keys)-1])].([]byte); ok { + return value + } + return nil +} + +// Put inserts a value into a key path. +func (db *QuickDB) Put(keys [][]byte, value []byte) { + db.Lock() + defer db.Unlock() + + // Build buckets all the way down the key path. + m := db.m + for _, key := range keys[:len(keys)-1] { + if _, ok := m[string(key)].([]byte); ok { + return // Keypath intersects with a simple value. Do nothing. + } + + if m[string(key)] == nil { + m[string(key)] = make(map[string]interface{}) + } + m = m[string(key)].(map[string]interface{}) + } + + // Insert value into the last key. + m[string(keys[len(keys)-1])] = value +} + +// Rand returns a random key path that points to a simple value. +func (db *QuickDB) Rand() [][]byte { + db.RLock() + defer db.RUnlock() + if len(db.m) == 0 { + return nil + } + var keys [][]byte + db.rand(db.m, &keys) + return keys +} + +func (db *QuickDB) rand(m map[string]interface{}, keys *[][]byte) { + i, index := 0, rand.Intn(len(m)) + for k, v := range m { + if i == index { + *keys = append(*keys, []byte(k)) + if v, ok := v.(map[string]interface{}); ok { + db.rand(v, keys) + } + return + } + i++ + } + panic("quickdb rand: out-of-range") +} + +// Copy copies the entire database. +func (db *QuickDB) Copy() *QuickDB { + db.RLock() + defer db.RUnlock() + return &QuickDB{m: db.copy(db.m)} +} + +func (db *QuickDB) copy(m map[string]interface{}) map[string]interface{} { + clone := make(map[string]interface{}, len(m)) + for k, v := range m { + switch v := v.(type) { + case map[string]interface{}: + clone[k] = db.copy(v) + default: + clone[k] = v + } + } + return clone +} + +func randKey() []byte { + var min, max = 1, 1024 + n := rand.Intn(max-min) + min + b := make([]byte, n) + for i := 0; i < n; i++ { + b[i] = byte(rand.Intn(255)) + } + return b +} + +func randKeys() [][]byte { + var keys [][]byte + var count = rand.Intn(2) + 2 + for i := 0; i < count; i++ { + keys = append(keys, randKey()) + } + return keys +} + +func randValue() []byte { + n := rand.Intn(8192) + b := make([]byte, n) + for i := 0; i < n; i++ { + b[i] = byte(rand.Intn(255)) + } + return b +} diff --git a/vendor/github.com/coreos/bbolt/tx.go b/vendor/github.com/coreos/bbolt/tx.go new file mode 100644 index 0000000..1cfb4cd --- /dev/null +++ b/vendor/github.com/coreos/bbolt/tx.go @@ -0,0 +1,682 @@ +package bolt + +import ( + "fmt" + "io" + "os" + "sort" + "strings" + "time" + "unsafe" +) + +// txid represents the internal transaction identifier. +type txid uint64 + +// Tx represents a read-only or read/write transaction on the database. +// Read-only transactions can be used for retrieving values for keys and creating cursors. +// Read/write transactions can create and remove buckets and create and remove keys. +// +// IMPORTANT: You must commit or rollback transactions when you are done with +// them. Pages can not be reclaimed by the writer until no more transactions +// are using them. A long running read transaction can cause the database to +// quickly grow. +type Tx struct { + writable bool + managed bool + db *DB + meta *meta + root Bucket + pages map[pgid]*page + stats TxStats + commitHandlers []func() + + // WriteFlag specifies the flag for write-related methods like WriteTo(). + // Tx opens the database file with the specified flag to copy the data. + // + // By default, the flag is unset, which works well for mostly in-memory + // workloads. For databases that are much larger than available RAM, + // set the flag to syscall.O_DIRECT to avoid trashing the page cache. + WriteFlag int +} + +// init initializes the transaction. +func (tx *Tx) init(db *DB) { + tx.db = db + tx.pages = nil + + // Copy the meta page since it can be changed by the writer. + tx.meta = &meta{} + db.meta().copy(tx.meta) + + // Copy over the root bucket. + tx.root = newBucket(tx) + tx.root.bucket = &bucket{} + *tx.root.bucket = tx.meta.root + + // Increment the transaction id and add a page cache for writable transactions. + if tx.writable { + tx.pages = make(map[pgid]*page) + tx.meta.txid += txid(1) + } +} + +// ID returns the transaction id. +func (tx *Tx) ID() int { + return int(tx.meta.txid) +} + +// DB returns a reference to the database that created the transaction. +func (tx *Tx) DB() *DB { + return tx.db +} + +// Size returns current database size in bytes as seen by this transaction. +func (tx *Tx) Size() int64 { + return int64(tx.meta.pgid) * int64(tx.db.pageSize) +} + +// Writable returns whether the transaction can perform write operations. +func (tx *Tx) Writable() bool { + return tx.writable +} + +// Cursor creates a cursor associated with the root bucket. +// All items in the cursor will return a nil value because all root bucket keys point to buckets. +// The cursor is only valid as long as the transaction is open. +// Do not use a cursor after the transaction is closed. +func (tx *Tx) Cursor() *Cursor { + return tx.root.Cursor() +} + +// Stats retrieves a copy of the current transaction statistics. +func (tx *Tx) Stats() TxStats { + return tx.stats +} + +// Bucket retrieves a bucket by name. +// Returns nil if the bucket does not exist. +// The bucket instance is only valid for the lifetime of the transaction. +func (tx *Tx) Bucket(name []byte) *Bucket { + return tx.root.Bucket(name) +} + +// CreateBucket creates a new bucket. +// Returns an error if the bucket already exists, if the bucket name is blank, or if the bucket name is too long. +// The bucket instance is only valid for the lifetime of the transaction. +func (tx *Tx) CreateBucket(name []byte) (*Bucket, error) { + return tx.root.CreateBucket(name) +} + +// CreateBucketIfNotExists creates a new bucket if it doesn't already exist. +// Returns an error if the bucket name is blank, or if the bucket name is too long. +// The bucket instance is only valid for the lifetime of the transaction. +func (tx *Tx) CreateBucketIfNotExists(name []byte) (*Bucket, error) { + return tx.root.CreateBucketIfNotExists(name) +} + +// DeleteBucket deletes a bucket. +// Returns an error if the bucket cannot be found or if the key represents a non-bucket value. +func (tx *Tx) DeleteBucket(name []byte) error { + return tx.root.DeleteBucket(name) +} + +// ForEach executes a function for each bucket in the root. +// If the provided function returns an error then the iteration is stopped and +// the error is returned to the caller. +func (tx *Tx) ForEach(fn func(name []byte, b *Bucket) error) error { + return tx.root.ForEach(func(k, v []byte) error { + if err := fn(k, tx.root.Bucket(k)); err != nil { + return err + } + return nil + }) +} + +// OnCommit adds a handler function to be executed after the transaction successfully commits. +func (tx *Tx) OnCommit(fn func()) { + tx.commitHandlers = append(tx.commitHandlers, fn) +} + +// Commit writes all changes to disk and updates the meta page. +// Returns an error if a disk write error occurs, or if Commit is +// called on a read-only transaction. +func (tx *Tx) Commit() error { + _assert(!tx.managed, "managed tx commit not allowed") + if tx.db == nil { + return ErrTxClosed + } else if !tx.writable { + return ErrTxNotWritable + } + + // TODO(benbjohnson): Use vectorized I/O to write out dirty pages. + + // Rebalance nodes which have had deletions. + var startTime = time.Now() + tx.root.rebalance() + if tx.stats.Rebalance > 0 { + tx.stats.RebalanceTime += time.Since(startTime) + } + + // spill data onto dirty pages. + startTime = time.Now() + if err := tx.root.spill(); err != nil { + tx.rollback() + return err + } + tx.stats.SpillTime += time.Since(startTime) + + // Free the old root bucket. + tx.meta.root.root = tx.root.root + + opgid := tx.meta.pgid + + // Free the freelist and allocate new pages for it. This will overestimate + // the size of the freelist but not underestimate the size (which would be bad). + tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist)) + p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1) + if err != nil { + tx.rollback() + return err + } + if err := tx.db.freelist.write(p); err != nil { + tx.rollback() + return err + } + tx.meta.freelist = p.id + + // If the high water mark has moved up then attempt to grow the database. + if tx.meta.pgid > opgid { + if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil { + tx.rollback() + return err + } + } + + // Write dirty pages to disk. + startTime = time.Now() + if err := tx.write(); err != nil { + tx.rollback() + return err + } + + // If strict mode is enabled then perform a consistency check. + // Only the first consistency error is reported in the panic. + if tx.db.StrictMode { + ch := tx.Check() + var errs []string + for { + err, ok := <-ch + if !ok { + break + } + errs = append(errs, err.Error()) + } + if len(errs) > 0 { + panic("check fail: " + strings.Join(errs, "\n")) + } + } + + // Write meta to disk. + if err := tx.writeMeta(); err != nil { + tx.rollback() + return err + } + tx.stats.WriteTime += time.Since(startTime) + + // Finalize the transaction. + tx.close() + + // Execute commit handlers now that the locks have been removed. + for _, fn := range tx.commitHandlers { + fn() + } + + return nil +} + +// Rollback closes the transaction and ignores all previous updates. Read-only +// transactions must be rolled back and not committed. +func (tx *Tx) Rollback() error { + _assert(!tx.managed, "managed tx rollback not allowed") + if tx.db == nil { + return ErrTxClosed + } + tx.rollback() + return nil +} + +func (tx *Tx) rollback() { + if tx.db == nil { + return + } + if tx.writable { + tx.db.freelist.rollback(tx.meta.txid) + tx.db.freelist.reload(tx.db.page(tx.db.meta().freelist)) + } + tx.close() +} + +func (tx *Tx) close() { + if tx.db == nil { + return + } + if tx.writable { + // Grab freelist stats. + var freelistFreeN = tx.db.freelist.free_count() + var freelistPendingN = tx.db.freelist.pending_count() + var freelistAlloc = tx.db.freelist.size() + + // Remove transaction ref & writer lock. + tx.db.rwtx = nil + tx.db.rwlock.Unlock() + + // Merge statistics. + tx.db.statlock.Lock() + tx.db.stats.FreePageN = freelistFreeN + tx.db.stats.PendingPageN = freelistPendingN + tx.db.stats.FreeAlloc = (freelistFreeN + freelistPendingN) * tx.db.pageSize + tx.db.stats.FreelistInuse = freelistAlloc + tx.db.stats.TxStats.add(&tx.stats) + tx.db.statlock.Unlock() + } else { + tx.db.removeTx(tx) + } + + // Clear all references. + tx.db = nil + tx.meta = nil + tx.root = Bucket{tx: tx} + tx.pages = nil +} + +// Copy writes the entire database to a writer. +// This function exists for backwards compatibility. Use WriteTo() instead. +func (tx *Tx) Copy(w io.Writer) error { + _, err := tx.WriteTo(w) + return err +} + +// WriteTo writes the entire database to a writer. +// If err == nil then exactly tx.Size() bytes will be written into the writer. +func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { + // Attempt to open reader with WriteFlag + f, err := os.OpenFile(tx.db.path, os.O_RDONLY|tx.WriteFlag, 0) + if err != nil { + return 0, err + } + defer func() { _ = f.Close() }() + + // Generate a meta page. We use the same page data for both meta pages. + buf := make([]byte, tx.db.pageSize) + page := (*page)(unsafe.Pointer(&buf[0])) + page.flags = metaPageFlag + *page.meta() = *tx.meta + + // Write meta 0. + page.id = 0 + page.meta().checksum = page.meta().sum64() + nn, err := w.Write(buf) + n += int64(nn) + if err != nil { + return n, fmt.Errorf("meta 0 copy: %s", err) + } + + // Write meta 1 with a lower transaction id. + page.id = 1 + page.meta().txid -= 1 + page.meta().checksum = page.meta().sum64() + nn, err = w.Write(buf) + n += int64(nn) + if err != nil { + return n, fmt.Errorf("meta 1 copy: %s", err) + } + + // Move past the meta pages in the file. + if _, err := f.Seek(int64(tx.db.pageSize*2), os.SEEK_SET); err != nil { + return n, fmt.Errorf("seek: %s", err) + } + + // Copy data pages. + wn, err := io.CopyN(w, f, tx.Size()-int64(tx.db.pageSize*2)) + n += wn + if err != nil { + return n, err + } + + return n, f.Close() +} + +// CopyFile copies the entire database to file at the given path. +// A reader transaction is maintained during the copy so it is safe to continue +// using the database while a copy is in progress. +func (tx *Tx) CopyFile(path string, mode os.FileMode) error { + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) + if err != nil { + return err + } + + err = tx.Copy(f) + if err != nil { + _ = f.Close() + return err + } + return f.Close() +} + +// Check performs several consistency checks on the database for this transaction. +// An error is returned if any inconsistency is found. +// +// It can be safely run concurrently on a writable transaction. However, this +// incurs a high cost for large databases and databases with a lot of subbuckets +// because of caching. This overhead can be removed if running on a read-only +// transaction, however, it is not safe to execute other writer transactions at +// the same time. +func (tx *Tx) Check() <-chan error { + ch := make(chan error) + go tx.check(ch) + return ch +} + +func (tx *Tx) check(ch chan error) { + // Check if any pages are double freed. + freed := make(map[pgid]bool) + for _, id := range tx.db.freelist.all() { + if freed[id] { + ch <- fmt.Errorf("page %d: already freed", id) + } + freed[id] = true + } + + // Track every reachable page. + reachable := make(map[pgid]*page) + reachable[0] = tx.page(0) // meta0 + reachable[1] = tx.page(1) // meta1 + for i := uint32(0); i <= tx.page(tx.meta.freelist).overflow; i++ { + reachable[tx.meta.freelist+pgid(i)] = tx.page(tx.meta.freelist) + } + + // Recursively check buckets. + tx.checkBucket(&tx.root, reachable, freed, ch) + + // Ensure all pages below high water mark are either reachable or freed. + for i := pgid(0); i < tx.meta.pgid; i++ { + _, isReachable := reachable[i] + if !isReachable && !freed[i] { + ch <- fmt.Errorf("page %d: unreachable unfreed", int(i)) + } + } + + // Close the channel to signal completion. + close(ch) +} + +func (tx *Tx) checkBucket(b *Bucket, reachable map[pgid]*page, freed map[pgid]bool, ch chan error) { + // Ignore inline buckets. + if b.root == 0 { + return + } + + // Check every page used by this bucket. + b.tx.forEachPage(b.root, 0, func(p *page, _ int) { + if p.id > tx.meta.pgid { + ch <- fmt.Errorf("page %d: out of bounds: %d", int(p.id), int(b.tx.meta.pgid)) + } + + // Ensure each page is only referenced once. + for i := pgid(0); i <= pgid(p.overflow); i++ { + var id = p.id + i + if _, ok := reachable[id]; ok { + ch <- fmt.Errorf("page %d: multiple references", int(id)) + } + reachable[id] = p + } + + // We should only encounter un-freed leaf and branch pages. + if freed[p.id] { + ch <- fmt.Errorf("page %d: reachable freed", int(p.id)) + } else if (p.flags&branchPageFlag) == 0 && (p.flags&leafPageFlag) == 0 { + ch <- fmt.Errorf("page %d: invalid type: %s", int(p.id), p.typ()) + } + }) + + // Check each bucket within this bucket. + _ = b.ForEach(func(k, v []byte) error { + if child := b.Bucket(k); child != nil { + tx.checkBucket(child, reachable, freed, ch) + } + return nil + }) +} + +// allocate returns a contiguous block of memory starting at a given page. +func (tx *Tx) allocate(count int) (*page, error) { + p, err := tx.db.allocate(count) + if err != nil { + return nil, err + } + + // Save to our page cache. + tx.pages[p.id] = p + + // Update statistics. + tx.stats.PageCount++ + tx.stats.PageAlloc += count * tx.db.pageSize + + return p, nil +} + +// write writes any dirty pages to disk. +func (tx *Tx) write() error { + // Sort pages by id. + pages := make(pages, 0, len(tx.pages)) + for _, p := range tx.pages { + pages = append(pages, p) + } + // Clear out page cache early. + tx.pages = make(map[pgid]*page) + sort.Sort(pages) + + // Write pages to disk in order. + for _, p := range pages { + size := (int(p.overflow) + 1) * tx.db.pageSize + offset := int64(p.id) * int64(tx.db.pageSize) + + // Write out page in "max allocation" sized chunks. + ptr := (*[maxAllocSize]byte)(unsafe.Pointer(p)) + for { + // Limit our write to our max allocation size. + sz := size + if sz > maxAllocSize-1 { + sz = maxAllocSize - 1 + } + + // Write chunk to disk. + buf := ptr[:sz] + if _, err := tx.db.ops.writeAt(buf, offset); err != nil { + return err + } + + // Update statistics. + tx.stats.Write++ + + // Exit inner for loop if we've written all the chunks. + size -= sz + if size == 0 { + break + } + + // Otherwise move offset forward and move pointer to next chunk. + offset += int64(sz) + ptr = (*[maxAllocSize]byte)(unsafe.Pointer(&ptr[sz])) + } + } + + // Ignore file sync if flag is set on DB. + if !tx.db.NoSync || IgnoreNoSync { + if err := fdatasync(tx.db); err != nil { + return err + } + } + + // Put small pages back to page pool. + for _, p := range pages { + // Ignore page sizes over 1 page. + // These are allocated using make() instead of the page pool. + if int(p.overflow) != 0 { + continue + } + + buf := (*[maxAllocSize]byte)(unsafe.Pointer(p))[:tx.db.pageSize] + + // See https://go.googlesource.com/go/+/f03c9202c43e0abb130669852082117ca50aa9b1 + for i := range buf { + buf[i] = 0 + } + tx.db.pagePool.Put(buf) + } + + return nil +} + +// writeMeta writes the meta to the disk. +func (tx *Tx) writeMeta() error { + // Create a temporary buffer for the meta page. + buf := make([]byte, tx.db.pageSize) + p := tx.db.pageInBuffer(buf, 0) + tx.meta.write(p) + + // Write the meta page to file. + if _, err := tx.db.ops.writeAt(buf, int64(p.id)*int64(tx.db.pageSize)); err != nil { + return err + } + if !tx.db.NoSync || IgnoreNoSync { + if err := fdatasync(tx.db); err != nil { + return err + } + } + + // Update statistics. + tx.stats.Write++ + + return nil +} + +// page returns a reference to the page with a given id. +// If page has been written to then a temporary buffered page is returned. +func (tx *Tx) page(id pgid) *page { + // Check the dirty pages first. + if tx.pages != nil { + if p, ok := tx.pages[id]; ok { + return p + } + } + + // Otherwise return directly from the mmap. + return tx.db.page(id) +} + +// forEachPage iterates over every page within a given page and executes a function. +func (tx *Tx) forEachPage(pgid pgid, depth int, fn func(*page, int)) { + p := tx.page(pgid) + + // Execute function. + fn(p, depth) + + // Recursively loop over children. + if (p.flags & branchPageFlag) != 0 { + for i := 0; i < int(p.count); i++ { + elem := p.branchPageElement(uint16(i)) + tx.forEachPage(elem.pgid, depth+1, fn) + } + } +} + +// Page returns page information for a given page number. +// This is only safe for concurrent use when used by a writable transaction. +func (tx *Tx) Page(id int) (*PageInfo, error) { + if tx.db == nil { + return nil, ErrTxClosed + } else if pgid(id) >= tx.meta.pgid { + return nil, nil + } + + // Build the page info. + p := tx.db.page(pgid(id)) + info := &PageInfo{ + ID: id, + Count: int(p.count), + OverflowCount: int(p.overflow), + } + + // Determine the type (or if it's free). + if tx.db.freelist.freed(pgid(id)) { + info.Type = "free" + } else { + info.Type = p.typ() + } + + return info, nil +} + +// TxStats represents statistics about the actions performed by the transaction. +type TxStats struct { + // Page statistics. + PageCount int // number of page allocations + PageAlloc int // total bytes allocated + + // Cursor statistics. + CursorCount int // number of cursors created + + // Node statistics + NodeCount int // number of node allocations + NodeDeref int // number of node dereferences + + // Rebalance statistics. + Rebalance int // number of node rebalances + RebalanceTime time.Duration // total time spent rebalancing + + // Split/Spill statistics. + Split int // number of nodes split + Spill int // number of nodes spilled + SpillTime time.Duration // total time spent spilling + + // Write statistics. + Write int // number of writes performed + WriteTime time.Duration // total time spent writing to disk +} + +func (s *TxStats) add(other *TxStats) { + s.PageCount += other.PageCount + s.PageAlloc += other.PageAlloc + s.CursorCount += other.CursorCount + s.NodeCount += other.NodeCount + s.NodeDeref += other.NodeDeref + s.Rebalance += other.Rebalance + s.RebalanceTime += other.RebalanceTime + s.Split += other.Split + s.Spill += other.Spill + s.SpillTime += other.SpillTime + s.Write += other.Write + s.WriteTime += other.WriteTime +} + +// Sub calculates and returns the difference between two sets of transaction stats. +// This is useful when obtaining stats at two different points and time and +// you need the performance counters that occurred within that time span. +func (s *TxStats) Sub(other *TxStats) TxStats { + var diff TxStats + diff.PageCount = s.PageCount - other.PageCount + diff.PageAlloc = s.PageAlloc - other.PageAlloc + diff.CursorCount = s.CursorCount - other.CursorCount + diff.NodeCount = s.NodeCount - other.NodeCount + diff.NodeDeref = s.NodeDeref - other.NodeDeref + diff.Rebalance = s.Rebalance - other.Rebalance + diff.RebalanceTime = s.RebalanceTime - other.RebalanceTime + diff.Split = s.Split - other.Split + diff.Spill = s.Spill - other.Spill + diff.SpillTime = s.SpillTime - other.SpillTime + diff.Write = s.Write - other.Write + diff.WriteTime = s.WriteTime - other.WriteTime + return diff +} diff --git a/vendor/github.com/coreos/bbolt/tx_test.go b/vendor/github.com/coreos/bbolt/tx_test.go new file mode 100644 index 0000000..2201e79 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/tx_test.go @@ -0,0 +1,716 @@ +package bolt_test + +import ( + "bytes" + "errors" + "fmt" + "log" + "os" + "testing" + + "github.com/boltdb/bolt" +) + +// Ensure that committing a closed transaction returns an error. +func TestTx_Commit_ErrTxClosed(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + tx, err := db.Begin(true) + if err != nil { + t.Fatal(err) + } + + if _, err := tx.CreateBucket([]byte("foo")); err != nil { + t.Fatal(err) + } + + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + + if err := tx.Commit(); err != bolt.ErrTxClosed { + t.Fatalf("unexpected error: %s", err) + } +} + +// Ensure that rolling back a closed transaction returns an error. +func TestTx_Rollback_ErrTxClosed(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + tx, err := db.Begin(true) + if err != nil { + t.Fatal(err) + } + + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + if err := tx.Rollback(); err != bolt.ErrTxClosed { + t.Fatalf("unexpected error: %s", err) + } +} + +// Ensure that committing a read-only transaction returns an error. +func TestTx_Commit_ErrTxNotWritable(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + tx, err := db.Begin(false) + if err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != bolt.ErrTxNotWritable { + t.Fatal(err) + } +} + +// Ensure that a transaction can retrieve a cursor on the root bucket. +func TestTx_Cursor(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + + if _, err := tx.CreateBucket([]byte("woojits")); err != nil { + t.Fatal(err) + } + + c := tx.Cursor() + if k, v := c.First(); !bytes.Equal(k, []byte("widgets")) { + t.Fatalf("unexpected key: %v", k) + } else if v != nil { + t.Fatalf("unexpected value: %v", v) + } + + if k, v := c.Next(); !bytes.Equal(k, []byte("woojits")) { + t.Fatalf("unexpected key: %v", k) + } else if v != nil { + t.Fatalf("unexpected value: %v", v) + } + + if k, v := c.Next(); k != nil { + t.Fatalf("unexpected key: %v", k) + } else if v != nil { + t.Fatalf("unexpected value: %v", k) + } + + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that creating a bucket with a read-only transaction returns an error. +func TestTx_CreateBucket_ErrTxNotWritable(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.View(func(tx *bolt.Tx) error { + _, err := tx.CreateBucket([]byte("foo")) + if err != bolt.ErrTxNotWritable { + t.Fatalf("unexpected error: %s", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that creating a bucket on a closed transaction returns an error. +func TestTx_CreateBucket_ErrTxClosed(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + tx, err := db.Begin(true) + if err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + + if _, err := tx.CreateBucket([]byte("foo")); err != bolt.ErrTxClosed { + t.Fatalf("unexpected error: %s", err) + } +} + +// Ensure that a Tx can retrieve a bucket. +func TestTx_Bucket(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + if tx.Bucket([]byte("widgets")) == nil { + t.Fatal("expected bucket") + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a Tx retrieving a non-existent key returns nil. +func TestTx_Get_NotFound(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + if b.Get([]byte("no_such_key")) != nil { + t.Fatal("expected nil value") + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a bucket can be created and retrieved. +func TestTx_CreateBucket(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + // Create a bucket. + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } else if b == nil { + t.Fatal("expected bucket") + } + return nil + }); err != nil { + t.Fatal(err) + } + + // Read the bucket through a separate transaction. + if err := db.View(func(tx *bolt.Tx) error { + if tx.Bucket([]byte("widgets")) == nil { + t.Fatal("expected bucket") + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a bucket can be created if it doesn't already exist. +func TestTx_CreateBucketIfNotExists(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + // Create bucket. + if b, err := tx.CreateBucketIfNotExists([]byte("widgets")); err != nil { + t.Fatal(err) + } else if b == nil { + t.Fatal("expected bucket") + } + + // Create bucket again. + if b, err := tx.CreateBucketIfNotExists([]byte("widgets")); err != nil { + t.Fatal(err) + } else if b == nil { + t.Fatal("expected bucket") + } + + return nil + }); err != nil { + t.Fatal(err) + } + + // Read the bucket through a separate transaction. + if err := db.View(func(tx *bolt.Tx) error { + if tx.Bucket([]byte("widgets")) == nil { + t.Fatal("expected bucket") + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure transaction returns an error if creating an unnamed bucket. +func TestTx_CreateBucketIfNotExists_ErrBucketNameRequired(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucketIfNotExists([]byte{}); err != bolt.ErrBucketNameRequired { + t.Fatalf("unexpected error: %s", err) + } + + if _, err := tx.CreateBucketIfNotExists(nil); err != bolt.ErrBucketNameRequired { + t.Fatalf("unexpected error: %s", err) + } + + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a bucket cannot be created twice. +func TestTx_CreateBucket_ErrBucketExists(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + // Create a bucket. + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + // Create the same bucket again. + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket([]byte("widgets")); err != bolt.ErrBucketExists { + t.Fatalf("unexpected error: %s", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a bucket is created with a non-blank name. +func TestTx_CreateBucket_ErrBucketNameRequired(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucket(nil); err != bolt.ErrBucketNameRequired { + t.Fatalf("unexpected error: %s", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that a bucket can be deleted. +func TestTx_DeleteBucket(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + // Create a bucket and add a value. + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + // Delete the bucket and make sure we can't get the value. + if err := db.Update(func(tx *bolt.Tx) error { + if err := tx.DeleteBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + if tx.Bucket([]byte("widgets")) != nil { + t.Fatal("unexpected bucket") + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.Update(func(tx *bolt.Tx) error { + // Create the bucket again and make sure there's not a phantom value. + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if v := b.Get([]byte("foo")); v != nil { + t.Fatalf("unexpected phantom value: %v", v) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that deleting a bucket on a closed transaction returns an error. +func TestTx_DeleteBucket_ErrTxClosed(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + tx, err := db.Begin(true) + if err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + if err := tx.DeleteBucket([]byte("foo")); err != bolt.ErrTxClosed { + t.Fatalf("unexpected error: %s", err) + } +} + +// Ensure that deleting a bucket with a read-only transaction returns an error. +func TestTx_DeleteBucket_ReadOnly(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.View(func(tx *bolt.Tx) error { + if err := tx.DeleteBucket([]byte("foo")); err != bolt.ErrTxNotWritable { + t.Fatalf("unexpected error: %s", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that nothing happens when deleting a bucket that doesn't exist. +func TestTx_DeleteBucket_NotFound(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + if err := tx.DeleteBucket([]byte("widgets")); err != bolt.ErrBucketNotFound { + t.Fatalf("unexpected error: %s", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that no error is returned when a tx.ForEach function does not return +// an error. +func TestTx_ForEach_NoError(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + + if err := tx.ForEach(func(name []byte, b *bolt.Bucket) error { + return nil + }); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that an error is returned when a tx.ForEach function returns an error. +func TestTx_ForEach_WithError(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + + marker := errors.New("marker") + if err := tx.ForEach(func(name []byte, b *bolt.Bucket) error { + return marker + }); err != marker { + t.Fatalf("unexpected error: %s", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// Ensure that Tx commit handlers are called after a transaction successfully commits. +func TestTx_OnCommit(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + var x int + if err := db.Update(func(tx *bolt.Tx) error { + tx.OnCommit(func() { x += 1 }) + tx.OnCommit(func() { x += 2 }) + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } else if x != 3 { + t.Fatalf("unexpected x: %d", x) + } +} + +// Ensure that Tx commit handlers are NOT called after a transaction rolls back. +func TestTx_OnCommit_Rollback(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + var x int + if err := db.Update(func(tx *bolt.Tx) error { + tx.OnCommit(func() { x += 1 }) + tx.OnCommit(func() { x += 2 }) + if _, err := tx.CreateBucket([]byte("widgets")); err != nil { + t.Fatal(err) + } + return errors.New("rollback this commit") + }); err == nil || err.Error() != "rollback this commit" { + t.Fatalf("unexpected error: %s", err) + } else if x != 0 { + t.Fatalf("unexpected x: %d", x) + } +} + +// Ensure that the database can be copied to a file path. +func TestTx_CopyFile(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + path := tempfile() + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("baz"), []byte("bat")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.View(func(tx *bolt.Tx) error { + return tx.CopyFile(path, 0600) + }); err != nil { + t.Fatal(err) + } + + db2, err := bolt.Open(path, 0600, nil) + if err != nil { + t.Fatal(err) + } + + if err := db2.View(func(tx *bolt.Tx) error { + if v := tx.Bucket([]byte("widgets")).Get([]byte("foo")); !bytes.Equal(v, []byte("bar")) { + t.Fatalf("unexpected value: %v", v) + } + if v := tx.Bucket([]byte("widgets")).Get([]byte("baz")); !bytes.Equal(v, []byte("bat")) { + t.Fatalf("unexpected value: %v", v) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db2.Close(); err != nil { + t.Fatal(err) + } +} + +type failWriterError struct{} + +func (failWriterError) Error() string { + return "error injected for tests" +} + +type failWriter struct { + // fail after this many bytes + After int +} + +func (f *failWriter) Write(p []byte) (n int, err error) { + n = len(p) + if n > f.After { + n = f.After + err = failWriterError{} + } + f.After -= n + return n, err +} + +// Ensure that Copy handles write errors right. +func TestTx_CopyFile_Error_Meta(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("baz"), []byte("bat")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.View(func(tx *bolt.Tx) error { + return tx.Copy(&failWriter{}) + }); err == nil || err.Error() != "meta 0 copy: error injected for tests" { + t.Fatalf("unexpected error: %v", err) + } +} + +// Ensure that Copy handles write errors right. +func TestTx_CopyFile_Error_Normal(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + t.Fatal(err) + } + if err := b.Put([]byte("baz"), []byte("bat")); err != nil { + t.Fatal(err) + } + return nil + }); err != nil { + t.Fatal(err) + } + + if err := db.View(func(tx *bolt.Tx) error { + return tx.Copy(&failWriter{3 * db.Info().PageSize}) + }); err == nil || err.Error() != "error injected for tests" { + t.Fatalf("unexpected error: %v", err) + } +} + +func ExampleTx_Rollback() { + // Open the database. + db, err := bolt.Open(tempfile(), 0666, nil) + if err != nil { + log.Fatal(err) + } + defer os.Remove(db.Path()) + + // Create a bucket. + if err := db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucket([]byte("widgets")) + return err + }); err != nil { + log.Fatal(err) + } + + // Set a value for a key. + if err := db.Update(func(tx *bolt.Tx) error { + return tx.Bucket([]byte("widgets")).Put([]byte("foo"), []byte("bar")) + }); err != nil { + log.Fatal(err) + } + + // Update the key but rollback the transaction so it never saves. + tx, err := db.Begin(true) + if err != nil { + log.Fatal(err) + } + b := tx.Bucket([]byte("widgets")) + if err := b.Put([]byte("foo"), []byte("baz")); err != nil { + log.Fatal(err) + } + if err := tx.Rollback(); err != nil { + log.Fatal(err) + } + + // Ensure that our original value is still set. + if err := db.View(func(tx *bolt.Tx) error { + value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) + fmt.Printf("The value for 'foo' is still: %s\n", value) + return nil + }); err != nil { + log.Fatal(err) + } + + // Close database to release file lock. + if err := db.Close(); err != nil { + log.Fatal(err) + } + + // Output: + // The value for 'foo' is still: bar +} + +func ExampleTx_CopyFile() { + // Open the database. + db, err := bolt.Open(tempfile(), 0666, nil) + if err != nil { + log.Fatal(err) + } + defer os.Remove(db.Path()) + + // Create a bucket and a key. + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + if err != nil { + return err + } + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { + return err + } + return nil + }); err != nil { + log.Fatal(err) + } + + // Copy the database to another file. + toFile := tempfile() + if err := db.View(func(tx *bolt.Tx) error { + return tx.CopyFile(toFile, 0666) + }); err != nil { + log.Fatal(err) + } + defer os.Remove(toFile) + + // Open the cloned database. + db2, err := bolt.Open(toFile, 0666, nil) + if err != nil { + log.Fatal(err) + } + + // Ensure that the key exists in the copy. + if err := db2.View(func(tx *bolt.Tx) error { + value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) + fmt.Printf("The value for 'foo' in the clone is: %s\n", value) + return nil + }); err != nil { + log.Fatal(err) + } + + // Close database to release file lock. + if err := db.Close(); err != nil { + log.Fatal(err) + } + + if err := db2.Close(); err != nil { + log.Fatal(err) + } + + // Output: + // The value for 'foo' in the clone is: bar +} diff --git a/vendor/github.com/dustin/go-humanize/.gitignore b/vendor/github.com/dustin/go-humanize/.gitignore new file mode 100644 index 0000000..05b4051 --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/.gitignore @@ -0,0 +1,6 @@ +#* +*.[568] +*.a +*~ +[568].out +_* diff --git a/vendor/github.com/dustin/go-humanize/.travis.yml b/vendor/github.com/dustin/go-humanize/.travis.yml new file mode 100644 index 0000000..ba95cdd --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/.travis.yml @@ -0,0 +1,21 @@ +sudo: false +language: go +go: + - 1.3.x + - 1.5.x + - 1.6.x + - 1.7.x + - 1.8.x + - 1.9.x + - master +matrix: + allow_failures: + - go: master + fast_finish: true +install: + - # Do nothing. This is needed to prevent default install action "go get -t -v ./..." from happening here (we want it to happen inside script step). +script: + - go get -t -v ./... + - diff -u <(echo -n) <(gofmt -d -s .) + - go tool vet . + - go test -v -race ./... diff --git a/vendor/github.com/dustin/go-humanize/LICENSE b/vendor/github.com/dustin/go-humanize/LICENSE new file mode 100644 index 0000000..8d9a94a --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/LICENSE @@ -0,0 +1,21 @@ +Copyright (c) 2005-2008 Dustin Sallings + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + diff --git a/vendor/github.com/dustin/go-humanize/README.markdown b/vendor/github.com/dustin/go-humanize/README.markdown new file mode 100644 index 0000000..f69d3c8 --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/README.markdown @@ -0,0 +1,92 @@ +# Humane Units [![Build Status](https://travis-ci.org/dustin/go-humanize.svg?branch=master)](https://travis-ci.org/dustin/go-humanize) [![GoDoc](https://godoc.org/github.com/dustin/go-humanize?status.svg)](https://godoc.org/github.com/dustin/go-humanize) + +Just a few functions for helping humanize times and sizes. + +`go get` it as `github.com/dustin/go-humanize`, import it as +`"github.com/dustin/go-humanize"`, use it as `humanize`. + +See [godoc](https://godoc.org/github.com/dustin/go-humanize) for +complete documentation. + +## Sizes + +This lets you take numbers like `82854982` and convert them to useful +strings like, `83 MB` or `79 MiB` (whichever you prefer). + +Example: + +```go +fmt.Printf("That file is %s.", humanize.Bytes(82854982)) // That file is 83 MB. +``` + +## Times + +This lets you take a `time.Time` and spit it out in relative terms. +For example, `12 seconds ago` or `3 days from now`. + +Example: + +```go +fmt.Printf("This was touched %s.", humanize.Time(someTimeInstance)) // This was touched 7 hours ago. +``` + +Thanks to Kyle Lemons for the time implementation from an IRC +conversation one day. It's pretty neat. + +## Ordinals + +From a [mailing list discussion][odisc] where a user wanted to be able +to label ordinals. + + 0 -> 0th + 1 -> 1st + 2 -> 2nd + 3 -> 3rd + 4 -> 4th + [...] + +Example: + +```go +fmt.Printf("You're my %s best friend.", humanize.Ordinal(193)) // You are my 193rd best friend. +``` + +## Commas + +Want to shove commas into numbers? Be my guest. + + 0 -> 0 + 100 -> 100 + 1000 -> 1,000 + 1000000000 -> 1,000,000,000 + -100000 -> -100,000 + +Example: + +```go +fmt.Printf("You owe $%s.\n", humanize.Comma(6582491)) // You owe $6,582,491. +``` + +## Ftoa + +Nicer float64 formatter that removes trailing zeros. + +```go +fmt.Printf("%f", 2.24) // 2.240000 +fmt.Printf("%s", humanize.Ftoa(2.24)) // 2.24 +fmt.Printf("%f", 2.0) // 2.000000 +fmt.Printf("%s", humanize.Ftoa(2.0)) // 2 +``` + +## SI notation + +Format numbers with [SI notation][sinotation]. + +Example: + +```go +humanize.SI(0.00000000223, "M") // 2.23 nM +``` + +[odisc]: https://groups.google.com/d/topic/golang-nuts/l8NhI74jl-4/discussion +[sinotation]: http://en.wikipedia.org/wiki/Metric_prefix diff --git a/vendor/github.com/dustin/go-humanize/big.go b/vendor/github.com/dustin/go-humanize/big.go new file mode 100644 index 0000000..f49dc33 --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/big.go @@ -0,0 +1,31 @@ +package humanize + +import ( + "math/big" +) + +// order of magnitude (to a max order) +func oomm(n, b *big.Int, maxmag int) (float64, int) { + mag := 0 + m := &big.Int{} + for n.Cmp(b) >= 0 { + n.DivMod(n, b, m) + mag++ + if mag == maxmag && maxmag >= 0 { + break + } + } + return float64(n.Int64()) + (float64(m.Int64()) / float64(b.Int64())), mag +} + +// total order of magnitude +// (same as above, but with no upper limit) +func oom(n, b *big.Int) (float64, int) { + mag := 0 + m := &big.Int{} + for n.Cmp(b) >= 0 { + n.DivMod(n, b, m) + mag++ + } + return float64(n.Int64()) + (float64(m.Int64()) / float64(b.Int64())), mag +} diff --git a/vendor/github.com/dustin/go-humanize/bigbytes.go b/vendor/github.com/dustin/go-humanize/bigbytes.go new file mode 100644 index 0000000..1a2bf61 --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/bigbytes.go @@ -0,0 +1,173 @@ +package humanize + +import ( + "fmt" + "math/big" + "strings" + "unicode" +) + +var ( + bigIECExp = big.NewInt(1024) + + // BigByte is one byte in bit.Ints + BigByte = big.NewInt(1) + // BigKiByte is 1,024 bytes in bit.Ints + BigKiByte = (&big.Int{}).Mul(BigByte, bigIECExp) + // BigMiByte is 1,024 k bytes in bit.Ints + BigMiByte = (&big.Int{}).Mul(BigKiByte, bigIECExp) + // BigGiByte is 1,024 m bytes in bit.Ints + BigGiByte = (&big.Int{}).Mul(BigMiByte, bigIECExp) + // BigTiByte is 1,024 g bytes in bit.Ints + BigTiByte = (&big.Int{}).Mul(BigGiByte, bigIECExp) + // BigPiByte is 1,024 t bytes in bit.Ints + BigPiByte = (&big.Int{}).Mul(BigTiByte, bigIECExp) + // BigEiByte is 1,024 p bytes in bit.Ints + BigEiByte = (&big.Int{}).Mul(BigPiByte, bigIECExp) + // BigZiByte is 1,024 e bytes in bit.Ints + BigZiByte = (&big.Int{}).Mul(BigEiByte, bigIECExp) + // BigYiByte is 1,024 z bytes in bit.Ints + BigYiByte = (&big.Int{}).Mul(BigZiByte, bigIECExp) +) + +var ( + bigSIExp = big.NewInt(1000) + + // BigSIByte is one SI byte in big.Ints + BigSIByte = big.NewInt(1) + // BigKByte is 1,000 SI bytes in big.Ints + BigKByte = (&big.Int{}).Mul(BigSIByte, bigSIExp) + // BigMByte is 1,000 SI k bytes in big.Ints + BigMByte = (&big.Int{}).Mul(BigKByte, bigSIExp) + // BigGByte is 1,000 SI m bytes in big.Ints + BigGByte = (&big.Int{}).Mul(BigMByte, bigSIExp) + // BigTByte is 1,000 SI g bytes in big.Ints + BigTByte = (&big.Int{}).Mul(BigGByte, bigSIExp) + // BigPByte is 1,000 SI t bytes in big.Ints + BigPByte = (&big.Int{}).Mul(BigTByte, bigSIExp) + // BigEByte is 1,000 SI p bytes in big.Ints + BigEByte = (&big.Int{}).Mul(BigPByte, bigSIExp) + // BigZByte is 1,000 SI e bytes in big.Ints + BigZByte = (&big.Int{}).Mul(BigEByte, bigSIExp) + // BigYByte is 1,000 SI z bytes in big.Ints + BigYByte = (&big.Int{}).Mul(BigZByte, bigSIExp) +) + +var bigBytesSizeTable = map[string]*big.Int{ + "b": BigByte, + "kib": BigKiByte, + "kb": BigKByte, + "mib": BigMiByte, + "mb": BigMByte, + "gib": BigGiByte, + "gb": BigGByte, + "tib": BigTiByte, + "tb": BigTByte, + "pib": BigPiByte, + "pb": BigPByte, + "eib": BigEiByte, + "eb": BigEByte, + "zib": BigZiByte, + "zb": BigZByte, + "yib": BigYiByte, + "yb": BigYByte, + // Without suffix + "": BigByte, + "ki": BigKiByte, + "k": BigKByte, + "mi": BigMiByte, + "m": BigMByte, + "gi": BigGiByte, + "g": BigGByte, + "ti": BigTiByte, + "t": BigTByte, + "pi": BigPiByte, + "p": BigPByte, + "ei": BigEiByte, + "e": BigEByte, + "z": BigZByte, + "zi": BigZiByte, + "y": BigYByte, + "yi": BigYiByte, +} + +var ten = big.NewInt(10) + +func humanateBigBytes(s, base *big.Int, sizes []string) string { + if s.Cmp(ten) < 0 { + return fmt.Sprintf("%d B", s) + } + c := (&big.Int{}).Set(s) + val, mag := oomm(c, base, len(sizes)-1) + suffix := sizes[mag] + f := "%.0f %s" + if val < 10 { + f = "%.1f %s" + } + + return fmt.Sprintf(f, val, suffix) + +} + +// BigBytes produces a human readable representation of an SI size. +// +// See also: ParseBigBytes. +// +// BigBytes(82854982) -> 83 MB +func BigBytes(s *big.Int) string { + sizes := []string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"} + return humanateBigBytes(s, bigSIExp, sizes) +} + +// BigIBytes produces a human readable representation of an IEC size. +// +// See also: ParseBigBytes. +// +// BigIBytes(82854982) -> 79 MiB +func BigIBytes(s *big.Int) string { + sizes := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"} + return humanateBigBytes(s, bigIECExp, sizes) +} + +// ParseBigBytes parses a string representation of bytes into the number +// of bytes it represents. +// +// See also: BigBytes, BigIBytes. +// +// ParseBigBytes("42 MB") -> 42000000, nil +// ParseBigBytes("42 mib") -> 44040192, nil +func ParseBigBytes(s string) (*big.Int, error) { + lastDigit := 0 + hasComma := false + for _, r := range s { + if !(unicode.IsDigit(r) || r == '.' || r == ',') { + break + } + if r == ',' { + hasComma = true + } + lastDigit++ + } + + num := s[:lastDigit] + if hasComma { + num = strings.Replace(num, ",", "", -1) + } + + val := &big.Rat{} + _, err := fmt.Sscanf(num, "%f", val) + if err != nil { + return nil, err + } + + extra := strings.ToLower(strings.TrimSpace(s[lastDigit:])) + if m, ok := bigBytesSizeTable[extra]; ok { + mv := (&big.Rat{}).SetInt(m) + val.Mul(val, mv) + rv := &big.Int{} + rv.Div(val.Num(), val.Denom()) + return rv, nil + } + + return nil, fmt.Errorf("unhandled size name: %v", extra) +} diff --git a/vendor/github.com/dustin/go-humanize/bigbytes_test.go b/vendor/github.com/dustin/go-humanize/bigbytes_test.go new file mode 100644 index 0000000..236ad08 --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/bigbytes_test.go @@ -0,0 +1,220 @@ +package humanize + +import ( + "math/big" + "testing" +) + +func TestBigByteParsing(t *testing.T) { + tests := []struct { + in string + exp uint64 + }{ + {"42", 42}, + {"42MB", 42000000}, + {"42MiB", 44040192}, + {"42mb", 42000000}, + {"42mib", 44040192}, + {"42MIB", 44040192}, + {"42 MB", 42000000}, + {"42 MiB", 44040192}, + {"42 mb", 42000000}, + {"42 mib", 44040192}, + {"42 MIB", 44040192}, + {"42.5MB", 42500000}, + {"42.5MiB", 44564480}, + {"42.5 MB", 42500000}, + {"42.5 MiB", 44564480}, + // No need to say B + {"42M", 42000000}, + {"42Mi", 44040192}, + {"42m", 42000000}, + {"42mi", 44040192}, + {"42MI", 44040192}, + {"42 M", 42000000}, + {"42 Mi", 44040192}, + {"42 m", 42000000}, + {"42 mi", 44040192}, + {"42 MI", 44040192}, + {"42.5M", 42500000}, + {"42.5Mi", 44564480}, + {"42.5 M", 42500000}, + {"42.5 Mi", 44564480}, + {"1,005.03 MB", 1005030000}, + // Large testing, breaks when too much larger than + // this. + {"12.5 EB", uint64(12.5 * float64(EByte))}, + {"12.5 E", uint64(12.5 * float64(EByte))}, + {"12.5 EiB", uint64(12.5 * float64(EiByte))}, + } + + for _, p := range tests { + got, err := ParseBigBytes(p.in) + if err != nil { + t.Errorf("Couldn't parse %v: %v", p.in, err) + } else { + if got.Uint64() != p.exp { + t.Errorf("Expected %v for %v, got %v", + p.exp, p.in, got) + } + } + } +} + +func TestBigByteErrors(t *testing.T) { + got, err := ParseBigBytes("84 JB") + if err == nil { + t.Errorf("Expected error, got %v", got) + } + got, err = ParseBigBytes("") + if err == nil { + t.Errorf("Expected error parsing nothing") + } +} + +func bbyte(in uint64) string { + return BigBytes((&big.Int{}).SetUint64(in)) +} + +func bibyte(in uint64) string { + return BigIBytes((&big.Int{}).SetUint64(in)) +} + +func TestBigBytes(t *testing.T) { + testList{ + {"bytes(0)", bbyte(0), "0 B"}, + {"bytes(1)", bbyte(1), "1 B"}, + {"bytes(803)", bbyte(803), "803 B"}, + {"bytes(999)", bbyte(999), "999 B"}, + + {"bytes(1024)", bbyte(1024), "1.0 kB"}, + {"bytes(1MB - 1)", bbyte(MByte - Byte), "1000 kB"}, + + {"bytes(1MB)", bbyte(1024 * 1024), "1.0 MB"}, + {"bytes(1GB - 1K)", bbyte(GByte - KByte), "1000 MB"}, + + {"bytes(1GB)", bbyte(GByte), "1.0 GB"}, + {"bytes(1TB - 1M)", bbyte(TByte - MByte), "1000 GB"}, + + {"bytes(1TB)", bbyte(TByte), "1.0 TB"}, + {"bytes(1PB - 1T)", bbyte(PByte - TByte), "999 TB"}, + + {"bytes(1PB)", bbyte(PByte), "1.0 PB"}, + {"bytes(1PB - 1T)", bbyte(EByte - PByte), "999 PB"}, + + {"bytes(1EB)", bbyte(EByte), "1.0 EB"}, + // Overflows. + // {"bytes(1EB - 1P)", Bytes((KByte*EByte)-PByte), "1023EB"}, + + {"bytes(0)", bibyte(0), "0 B"}, + {"bytes(1)", bibyte(1), "1 B"}, + {"bytes(803)", bibyte(803), "803 B"}, + {"bytes(1023)", bibyte(1023), "1023 B"}, + + {"bytes(1024)", bibyte(1024), "1.0 KiB"}, + {"bytes(1MB - 1)", bibyte(MiByte - IByte), "1024 KiB"}, + + {"bytes(1MB)", bibyte(1024 * 1024), "1.0 MiB"}, + {"bytes(1GB - 1K)", bibyte(GiByte - KiByte), "1024 MiB"}, + + {"bytes(1GB)", bibyte(GiByte), "1.0 GiB"}, + {"bytes(1TB - 1M)", bibyte(TiByte - MiByte), "1024 GiB"}, + + {"bytes(1TB)", bibyte(TiByte), "1.0 TiB"}, + {"bytes(1PB - 1T)", bibyte(PiByte - TiByte), "1023 TiB"}, + + {"bytes(1PB)", bibyte(PiByte), "1.0 PiB"}, + {"bytes(1PB - 1T)", bibyte(EiByte - PiByte), "1023 PiB"}, + + {"bytes(1EiB)", bibyte(EiByte), "1.0 EiB"}, + // Overflows. + // {"bytes(1EB - 1P)", bibyte((KIByte*EIByte)-PiByte), "1023EB"}, + + {"bytes(5.5GiB)", bibyte(5.5 * GiByte), "5.5 GiB"}, + + {"bytes(5.5GB)", bbyte(5.5 * GByte), "5.5 GB"}, + }.validate(t) +} + +func TestVeryBigBytes(t *testing.T) { + b, _ := (&big.Int{}).SetString("15347691069326346944512", 10) + s := BigBytes(b) + if s != "15 ZB" { + t.Errorf("Expected 15 ZB, got %v", s) + } + s = BigIBytes(b) + if s != "13 ZiB" { + t.Errorf("Expected 13 ZiB, got %v", s) + } + + b, _ = (&big.Int{}).SetString("15716035654990179271180288", 10) + s = BigBytes(b) + if s != "16 YB" { + t.Errorf("Expected 16 YB, got %v", s) + } + s = BigIBytes(b) + if s != "13 YiB" { + t.Errorf("Expected 13 YiB, got %v", s) + } +} + +func TestVeryVeryBigBytes(t *testing.T) { + b, _ := (&big.Int{}).SetString("16093220510709943573688614912", 10) + s := BigBytes(b) + if s != "16093 YB" { + t.Errorf("Expected 16093 YB, got %v", s) + } + s = BigIBytes(b) + if s != "13312 YiB" { + t.Errorf("Expected 13312 YiB, got %v", s) + } +} + +func TestParseVeryBig(t *testing.T) { + tests := []struct { + in string + out string + }{ + {"16 ZB", "16000000000000000000000"}, + {"16 ZiB", "18889465931478580854784"}, + {"16.5 ZB", "16500000000000000000000"}, + {"16.5 ZiB", "19479761741837286506496"}, + {"16 Z", "16000000000000000000000"}, + {"16 Zi", "18889465931478580854784"}, + {"16.5 Z", "16500000000000000000000"}, + {"16.5 Zi", "19479761741837286506496"}, + + {"16 YB", "16000000000000000000000000"}, + {"16 YiB", "19342813113834066795298816"}, + {"16.5 YB", "16500000000000000000000000"}, + {"16.5 YiB", "19947276023641381382651904"}, + {"16 Y", "16000000000000000000000000"}, + {"16 Yi", "19342813113834066795298816"}, + {"16.5 Y", "16500000000000000000000000"}, + {"16.5 Yi", "19947276023641381382651904"}, + } + + for _, test := range tests { + x, err := ParseBigBytes(test.in) + if err != nil { + t.Errorf("Error parsing %q: %v", test.in, err) + continue + } + + if x.String() != test.out { + t.Errorf("Expected %q for %q, got %v", test.out, test.in, x) + } + } +} + +func BenchmarkParseBigBytes(b *testing.B) { + for i := 0; i < b.N; i++ { + ParseBigBytes("16.5 Z") + } +} + +func BenchmarkBigBytes(b *testing.B) { + for i := 0; i < b.N; i++ { + bibyte(16.5 * GByte) + } +} diff --git a/vendor/github.com/dustin/go-humanize/bytes.go b/vendor/github.com/dustin/go-humanize/bytes.go new file mode 100644 index 0000000..0b498f4 --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/bytes.go @@ -0,0 +1,143 @@ +package humanize + +import ( + "fmt" + "math" + "strconv" + "strings" + "unicode" +) + +// IEC Sizes. +// kibis of bits +const ( + Byte = 1 << (iota * 10) + KiByte + MiByte + GiByte + TiByte + PiByte + EiByte +) + +// SI Sizes. +const ( + IByte = 1 + KByte = IByte * 1000 + MByte = KByte * 1000 + GByte = MByte * 1000 + TByte = GByte * 1000 + PByte = TByte * 1000 + EByte = PByte * 1000 +) + +var bytesSizeTable = map[string]uint64{ + "b": Byte, + "kib": KiByte, + "kb": KByte, + "mib": MiByte, + "mb": MByte, + "gib": GiByte, + "gb": GByte, + "tib": TiByte, + "tb": TByte, + "pib": PiByte, + "pb": PByte, + "eib": EiByte, + "eb": EByte, + // Without suffix + "": Byte, + "ki": KiByte, + "k": KByte, + "mi": MiByte, + "m": MByte, + "gi": GiByte, + "g": GByte, + "ti": TiByte, + "t": TByte, + "pi": PiByte, + "p": PByte, + "ei": EiByte, + "e": EByte, +} + +func logn(n, b float64) float64 { + return math.Log(n) / math.Log(b) +} + +func humanateBytes(s uint64, base float64, sizes []string) string { + if s < 10 { + return fmt.Sprintf("%d B", s) + } + e := math.Floor(logn(float64(s), base)) + suffix := sizes[int(e)] + val := math.Floor(float64(s)/math.Pow(base, e)*10+0.5) / 10 + f := "%.0f %s" + if val < 10 { + f = "%.1f %s" + } + + return fmt.Sprintf(f, val, suffix) +} + +// Bytes produces a human readable representation of an SI size. +// +// See also: ParseBytes. +// +// Bytes(82854982) -> 83 MB +func Bytes(s uint64) string { + sizes := []string{"B", "kB", "MB", "GB", "TB", "PB", "EB"} + return humanateBytes(s, 1000, sizes) +} + +// IBytes produces a human readable representation of an IEC size. +// +// See also: ParseBytes. +// +// IBytes(82854982) -> 79 MiB +func IBytes(s uint64) string { + sizes := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB"} + return humanateBytes(s, 1024, sizes) +} + +// ParseBytes parses a string representation of bytes into the number +// of bytes it represents. +// +// See Also: Bytes, IBytes. +// +// ParseBytes("42 MB") -> 42000000, nil +// ParseBytes("42 mib") -> 44040192, nil +func ParseBytes(s string) (uint64, error) { + lastDigit := 0 + hasComma := false + for _, r := range s { + if !(unicode.IsDigit(r) || r == '.' || r == ',') { + break + } + if r == ',' { + hasComma = true + } + lastDigit++ + } + + num := s[:lastDigit] + if hasComma { + num = strings.Replace(num, ",", "", -1) + } + + f, err := strconv.ParseFloat(num, 64) + if err != nil { + return 0, err + } + + extra := strings.ToLower(strings.TrimSpace(s[lastDigit:])) + if m, ok := bytesSizeTable[extra]; ok { + f *= float64(m) + if f >= math.MaxUint64 { + return 0, fmt.Errorf("too large: %v", s) + } + return uint64(f), nil + } + + return 0, fmt.Errorf("unhandled size name: %v", extra) +} diff --git a/vendor/github.com/dustin/go-humanize/bytes_test.go b/vendor/github.com/dustin/go-humanize/bytes_test.go new file mode 100644 index 0000000..0bb811c --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/bytes_test.go @@ -0,0 +1,146 @@ +package humanize + +import ( + "testing" +) + +func TestByteParsing(t *testing.T) { + tests := []struct { + in string + exp uint64 + }{ + {"42", 42}, + {"42MB", 42000000}, + {"42MiB", 44040192}, + {"42mb", 42000000}, + {"42mib", 44040192}, + {"42MIB", 44040192}, + {"42 MB", 42000000}, + {"42 MiB", 44040192}, + {"42 mb", 42000000}, + {"42 mib", 44040192}, + {"42 MIB", 44040192}, + {"42.5MB", 42500000}, + {"42.5MiB", 44564480}, + {"42.5 MB", 42500000}, + {"42.5 MiB", 44564480}, + // No need to say B + {"42M", 42000000}, + {"42Mi", 44040192}, + {"42m", 42000000}, + {"42mi", 44040192}, + {"42MI", 44040192}, + {"42 M", 42000000}, + {"42 Mi", 44040192}, + {"42 m", 42000000}, + {"42 mi", 44040192}, + {"42 MI", 44040192}, + {"42.5M", 42500000}, + {"42.5Mi", 44564480}, + {"42.5 M", 42500000}, + {"42.5 Mi", 44564480}, + // Bug #42 + {"1,005.03 MB", 1005030000}, + // Large testing, breaks when too much larger than + // this. + {"12.5 EB", uint64(12.5 * float64(EByte))}, + {"12.5 E", uint64(12.5 * float64(EByte))}, + {"12.5 EiB", uint64(12.5 * float64(EiByte))}, + } + + for _, p := range tests { + got, err := ParseBytes(p.in) + if err != nil { + t.Errorf("Couldn't parse %v: %v", p.in, err) + } + if got != p.exp { + t.Errorf("Expected %v for %v, got %v", + p.exp, p.in, got) + } + } +} + +func TestByteErrors(t *testing.T) { + got, err := ParseBytes("84 JB") + if err == nil { + t.Errorf("Expected error, got %v", got) + } + got, err = ParseBytes("") + if err == nil { + t.Errorf("Expected error parsing nothing") + } + got, err = ParseBytes("16 EiB") + if err == nil { + t.Errorf("Expected error, got %v", got) + } +} + +func TestBytes(t *testing.T) { + testList{ + {"bytes(0)", Bytes(0), "0 B"}, + {"bytes(1)", Bytes(1), "1 B"}, + {"bytes(803)", Bytes(803), "803 B"}, + {"bytes(999)", Bytes(999), "999 B"}, + + {"bytes(1024)", Bytes(1024), "1.0 kB"}, + {"bytes(9999)", Bytes(9999), "10 kB"}, + {"bytes(1MB - 1)", Bytes(MByte - Byte), "1000 kB"}, + + {"bytes(1MB)", Bytes(1024 * 1024), "1.0 MB"}, + {"bytes(1GB - 1K)", Bytes(GByte - KByte), "1000 MB"}, + + {"bytes(1GB)", Bytes(GByte), "1.0 GB"}, + {"bytes(1TB - 1M)", Bytes(TByte - MByte), "1000 GB"}, + {"bytes(10MB)", Bytes(9999 * 1000), "10 MB"}, + + {"bytes(1TB)", Bytes(TByte), "1.0 TB"}, + {"bytes(1PB - 1T)", Bytes(PByte - TByte), "999 TB"}, + + {"bytes(1PB)", Bytes(PByte), "1.0 PB"}, + {"bytes(1PB - 1T)", Bytes(EByte - PByte), "999 PB"}, + + {"bytes(1EB)", Bytes(EByte), "1.0 EB"}, + // Overflows. + // {"bytes(1EB - 1P)", Bytes((KByte*EByte)-PByte), "1023EB"}, + + {"bytes(0)", IBytes(0), "0 B"}, + {"bytes(1)", IBytes(1), "1 B"}, + {"bytes(803)", IBytes(803), "803 B"}, + {"bytes(1023)", IBytes(1023), "1023 B"}, + + {"bytes(1024)", IBytes(1024), "1.0 KiB"}, + {"bytes(1MB - 1)", IBytes(MiByte - IByte), "1024 KiB"}, + + {"bytes(1MB)", IBytes(1024 * 1024), "1.0 MiB"}, + {"bytes(1GB - 1K)", IBytes(GiByte - KiByte), "1024 MiB"}, + + {"bytes(1GB)", IBytes(GiByte), "1.0 GiB"}, + {"bytes(1TB - 1M)", IBytes(TiByte - MiByte), "1024 GiB"}, + + {"bytes(1TB)", IBytes(TiByte), "1.0 TiB"}, + {"bytes(1PB - 1T)", IBytes(PiByte - TiByte), "1023 TiB"}, + + {"bytes(1PB)", IBytes(PiByte), "1.0 PiB"}, + {"bytes(1PB - 1T)", IBytes(EiByte - PiByte), "1023 PiB"}, + + {"bytes(1EiB)", IBytes(EiByte), "1.0 EiB"}, + // Overflows. + // {"bytes(1EB - 1P)", IBytes((KIByte*EIByte)-PiByte), "1023EB"}, + + {"bytes(5.5GiB)", IBytes(5.5 * GiByte), "5.5 GiB"}, + + {"bytes(5.5GB)", Bytes(5.5 * GByte), "5.5 GB"}, + }.validate(t) +} + +func BenchmarkParseBytes(b *testing.B) { + for i := 0; i < b.N; i++ { + ParseBytes("16.5 GB") + } +} + +func BenchmarkBytes(b *testing.B) { + for i := 0; i < b.N; i++ { + Bytes(16.5 * GByte) + } +} diff --git a/vendor/github.com/dustin/go-humanize/comma.go b/vendor/github.com/dustin/go-humanize/comma.go new file mode 100644 index 0000000..eb285cb --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/comma.go @@ -0,0 +1,108 @@ +package humanize + +import ( + "bytes" + "math" + "math/big" + "strconv" + "strings" +) + +// Comma produces a string form of the given number in base 10 with +// commas after every three orders of magnitude. +// +// e.g. Comma(834142) -> 834,142 +func Comma(v int64) string { + sign := "" + + // minin64 can't be negated to a usable value, so it has to be special cased. + if v == math.MinInt64 { + return "-9,223,372,036,854,775,808" + } + + if v < 0 { + sign = "-" + v = 0 - v + } + + parts := []string{"", "", "", "", "", "", ""} + j := len(parts) - 1 + + for v > 999 { + parts[j] = strconv.FormatInt(v%1000, 10) + switch len(parts[j]) { + case 2: + parts[j] = "0" + parts[j] + case 1: + parts[j] = "00" + parts[j] + } + v = v / 1000 + j-- + } + parts[j] = strconv.Itoa(int(v)) + return sign + strings.Join(parts[j:], ",") +} + +// Commaf produces a string form of the given number in base 10 with +// commas after every three orders of magnitude. +// +// e.g. Commaf(834142.32) -> 834,142.32 +func Commaf(v float64) string { + buf := &bytes.Buffer{} + if v < 0 { + buf.Write([]byte{'-'}) + v = 0 - v + } + + comma := []byte{','} + + parts := strings.Split(strconv.FormatFloat(v, 'f', -1, 64), ".") + pos := 0 + if len(parts[0])%3 != 0 { + pos += len(parts[0]) % 3 + buf.WriteString(parts[0][:pos]) + buf.Write(comma) + } + for ; pos < len(parts[0]); pos += 3 { + buf.WriteString(parts[0][pos : pos+3]) + buf.Write(comma) + } + buf.Truncate(buf.Len() - 1) + + if len(parts) > 1 { + buf.Write([]byte{'.'}) + buf.WriteString(parts[1]) + } + return buf.String() +} + +// BigComma produces a string form of the given big.Int in base 10 +// with commas after every three orders of magnitude. +func BigComma(b *big.Int) string { + sign := "" + if b.Sign() < 0 { + sign = "-" + b.Abs(b) + } + + athousand := big.NewInt(1000) + c := (&big.Int{}).Set(b) + _, m := oom(c, athousand) + parts := make([]string, m+1) + j := len(parts) - 1 + + mod := &big.Int{} + for b.Cmp(athousand) >= 0 { + b.DivMod(b, athousand, mod) + parts[j] = strconv.FormatInt(mod.Int64(), 10) + switch len(parts[j]) { + case 2: + parts[j] = "0" + parts[j] + case 1: + parts[j] = "00" + parts[j] + } + j-- + } + parts[j] = strconv.Itoa(int(b.Int64())) + return sign + strings.Join(parts[j:], ",") +} diff --git a/vendor/github.com/dustin/go-humanize/comma_test.go b/vendor/github.com/dustin/go-humanize/comma_test.go new file mode 100644 index 0000000..89daca5 --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/comma_test.go @@ -0,0 +1,136 @@ +package humanize + +import ( + "math" + "math/big" + "testing" +) + +func TestCommas(t *testing.T) { + testList{ + {"0", Comma(0), "0"}, + {"10", Comma(10), "10"}, + {"100", Comma(100), "100"}, + {"1,000", Comma(1000), "1,000"}, + {"10,000", Comma(10000), "10,000"}, + {"100,000", Comma(100000), "100,000"}, + {"10,000,000", Comma(10000000), "10,000,000"}, + {"10,100,000", Comma(10100000), "10,100,000"}, + {"10,010,000", Comma(10010000), "10,010,000"}, + {"10,001,000", Comma(10001000), "10,001,000"}, + {"123,456,789", Comma(123456789), "123,456,789"}, + {"maxint", Comma(9.223372e+18), "9,223,372,000,000,000,000"}, + {"math.maxint", Comma(math.MaxInt64), "9,223,372,036,854,775,807"}, + {"math.minint", Comma(math.MinInt64), "-9,223,372,036,854,775,808"}, + {"minint", Comma(-9.223372e+18), "-9,223,372,000,000,000,000"}, + {"-123,456,789", Comma(-123456789), "-123,456,789"}, + {"-10,100,000", Comma(-10100000), "-10,100,000"}, + {"-10,010,000", Comma(-10010000), "-10,010,000"}, + {"-10,001,000", Comma(-10001000), "-10,001,000"}, + {"-10,000,000", Comma(-10000000), "-10,000,000"}, + {"-100,000", Comma(-100000), "-100,000"}, + {"-10,000", Comma(-10000), "-10,000"}, + {"-1,000", Comma(-1000), "-1,000"}, + {"-100", Comma(-100), "-100"}, + {"-10", Comma(-10), "-10"}, + }.validate(t) +} + +func TestCommafs(t *testing.T) { + testList{ + {"0", Commaf(0), "0"}, + {"10.11", Commaf(10.11), "10.11"}, + {"100", Commaf(100), "100"}, + {"1,000", Commaf(1000), "1,000"}, + {"10,000", Commaf(10000), "10,000"}, + {"100,000", Commaf(100000), "100,000"}, + {"834,142.32", Commaf(834142.32), "834,142.32"}, + {"10,000,000", Commaf(10000000), "10,000,000"}, + {"10,100,000", Commaf(10100000), "10,100,000"}, + {"10,010,000", Commaf(10010000), "10,010,000"}, + {"10,001,000", Commaf(10001000), "10,001,000"}, + {"123,456,789", Commaf(123456789), "123,456,789"}, + {"maxf64", Commaf(math.MaxFloat64), "179,769,313,486,231,570,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000"}, + {"minf64", Commaf(math.SmallestNonzeroFloat64), "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005"}, + {"-123,456,789", Commaf(-123456789), "-123,456,789"}, + {"-10,100,000", Commaf(-10100000), "-10,100,000"}, + {"-10,010,000", Commaf(-10010000), "-10,010,000"}, + {"-10,001,000", Commaf(-10001000), "-10,001,000"}, + {"-10,000,000", Commaf(-10000000), "-10,000,000"}, + {"-100,000", Commaf(-100000), "-100,000"}, + {"-10,000", Commaf(-10000), "-10,000"}, + {"-1,000", Commaf(-1000), "-1,000"}, + {"-100.11", Commaf(-100.11), "-100.11"}, + {"-10", Commaf(-10), "-10"}, + }.validate(t) +} + +func BenchmarkCommas(b *testing.B) { + for i := 0; i < b.N; i++ { + Comma(1234567890) + } +} + +func BenchmarkCommaf(b *testing.B) { + for i := 0; i < b.N; i++ { + Commaf(1234567890.83584) + } +} + +func BenchmarkBigCommas(b *testing.B) { + for i := 0; i < b.N; i++ { + BigComma(big.NewInt(1234567890)) + } +} + +func bigComma(i int64) string { + return BigComma(big.NewInt(i)) +} + +func TestBigCommas(t *testing.T) { + testList{ + {"0", bigComma(0), "0"}, + {"10", bigComma(10), "10"}, + {"100", bigComma(100), "100"}, + {"1,000", bigComma(1000), "1,000"}, + {"10,000", bigComma(10000), "10,000"}, + {"100,000", bigComma(100000), "100,000"}, + {"10,000,000", bigComma(10000000), "10,000,000"}, + {"10,100,000", bigComma(10100000), "10,100,000"}, + {"10,010,000", bigComma(10010000), "10,010,000"}, + {"10,001,000", bigComma(10001000), "10,001,000"}, + {"123,456,789", bigComma(123456789), "123,456,789"}, + {"maxint", bigComma(9.223372e+18), "9,223,372,000,000,000,000"}, + {"minint", bigComma(-9.223372e+18), "-9,223,372,000,000,000,000"}, + {"-123,456,789", bigComma(-123456789), "-123,456,789"}, + {"-10,100,000", bigComma(-10100000), "-10,100,000"}, + {"-10,010,000", bigComma(-10010000), "-10,010,000"}, + {"-10,001,000", bigComma(-10001000), "-10,001,000"}, + {"-10,000,000", bigComma(-10000000), "-10,000,000"}, + {"-100,000", bigComma(-100000), "-100,000"}, + {"-10,000", bigComma(-10000), "-10,000"}, + {"-1,000", bigComma(-1000), "-1,000"}, + {"-100", bigComma(-100), "-100"}, + {"-10", bigComma(-10), "-10"}, + }.validate(t) +} + +func TestVeryBigCommas(t *testing.T) { + tests := []struct{ in, exp string }{ + { + "84889279597249724975972597249849757294578485", + "84,889,279,597,249,724,975,972,597,249,849,757,294,578,485", + }, + { + "-84889279597249724975972597249849757294578485", + "-84,889,279,597,249,724,975,972,597,249,849,757,294,578,485", + }, + } + for _, test := range tests { + n, _ := (&big.Int{}).SetString(test.in, 10) + got := BigComma(n) + if test.exp != got { + t.Errorf("Expected %q, got %q", test.exp, got) + } + } +} diff --git a/vendor/github.com/dustin/go-humanize/commaf.go b/vendor/github.com/dustin/go-humanize/commaf.go new file mode 100644 index 0000000..620690d --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/commaf.go @@ -0,0 +1,40 @@ +// +build go1.6 + +package humanize + +import ( + "bytes" + "math/big" + "strings" +) + +// BigCommaf produces a string form of the given big.Float in base 10 +// with commas after every three orders of magnitude. +func BigCommaf(v *big.Float) string { + buf := &bytes.Buffer{} + if v.Sign() < 0 { + buf.Write([]byte{'-'}) + v.Abs(v) + } + + comma := []byte{','} + + parts := strings.Split(v.Text('f', -1), ".") + pos := 0 + if len(parts[0])%3 != 0 { + pos += len(parts[0]) % 3 + buf.WriteString(parts[0][:pos]) + buf.Write(comma) + } + for ; pos < len(parts[0]); pos += 3 { + buf.WriteString(parts[0][pos : pos+3]) + buf.Write(comma) + } + buf.Truncate(buf.Len() - 1) + + if len(parts) > 1 { + buf.Write([]byte{'.'}) + buf.WriteString(parts[1]) + } + return buf.String() +} diff --git a/vendor/github.com/dustin/go-humanize/commaf_test.go b/vendor/github.com/dustin/go-humanize/commaf_test.go new file mode 100644 index 0000000..21f7f9e --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/commaf_test.go @@ -0,0 +1,44 @@ +// +build go1.6 + +package humanize + +import ( + "math" + "math/big" + "testing" +) + +func BenchmarkBigCommaf(b *testing.B) { + for i := 0; i < b.N; i++ { + Commaf(1234567890.83584) + } +} + +func TestBigCommafs(t *testing.T) { + testList{ + {"0", BigCommaf(big.NewFloat(0)), "0"}, + {"10.11", BigCommaf(big.NewFloat(10.11)), "10.11"}, + {"100", BigCommaf(big.NewFloat(100)), "100"}, + {"1,000", BigCommaf(big.NewFloat(1000)), "1,000"}, + {"10,000", BigCommaf(big.NewFloat(10000)), "10,000"}, + {"100,000", BigCommaf(big.NewFloat(100000)), "100,000"}, + {"834,142.32", BigCommaf(big.NewFloat(834142.32)), "834,142.32"}, + {"10,000,000", BigCommaf(big.NewFloat(10000000)), "10,000,000"}, + {"10,100,000", BigCommaf(big.NewFloat(10100000)), "10,100,000"}, + {"10,010,000", BigCommaf(big.NewFloat(10010000)), "10,010,000"}, + {"10,001,000", BigCommaf(big.NewFloat(10001000)), "10,001,000"}, + {"123,456,789", BigCommaf(big.NewFloat(123456789)), "123,456,789"}, + {"maxf64", BigCommaf(big.NewFloat(math.MaxFloat64)), "179,769,313,486,231,570,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000"}, + {"minf64", BigCommaf(big.NewFloat(math.SmallestNonzeroFloat64)), "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004940656458412465"}, + {"-123,456,789", BigCommaf(big.NewFloat(-123456789)), "-123,456,789"}, + {"-10,100,000", BigCommaf(big.NewFloat(-10100000)), "-10,100,000"}, + {"-10,010,000", BigCommaf(big.NewFloat(-10010000)), "-10,010,000"}, + {"-10,001,000", BigCommaf(big.NewFloat(-10001000)), "-10,001,000"}, + {"-10,000,000", BigCommaf(big.NewFloat(-10000000)), "-10,000,000"}, + {"-100,000", BigCommaf(big.NewFloat(-100000)), "-100,000"}, + {"-10,000", BigCommaf(big.NewFloat(-10000)), "-10,000"}, + {"-1,000", BigCommaf(big.NewFloat(-1000)), "-1,000"}, + {"-100.11", BigCommaf(big.NewFloat(-100.11)), "-100.11"}, + {"-10", BigCommaf(big.NewFloat(-10)), "-10"}, + }.validate(t) +} diff --git a/vendor/github.com/dustin/go-humanize/common_test.go b/vendor/github.com/dustin/go-humanize/common_test.go new file mode 100644 index 0000000..fc7db15 --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/common_test.go @@ -0,0 +1,18 @@ +package humanize + +import ( + "testing" +) + +type testList []struct { + name, got, exp string +} + +func (tl testList) validate(t *testing.T) { + for _, test := range tl { + if test.got != test.exp { + t.Errorf("On %v, expected '%v', but got '%v'", + test.name, test.exp, test.got) + } + } +} diff --git a/vendor/github.com/dustin/go-humanize/ftoa.go b/vendor/github.com/dustin/go-humanize/ftoa.go new file mode 100644 index 0000000..c76190b --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/ftoa.go @@ -0,0 +1,23 @@ +package humanize + +import "strconv" + +func stripTrailingZeros(s string) string { + offset := len(s) - 1 + for offset > 0 { + if s[offset] == '.' { + offset-- + break + } + if s[offset] != '0' { + break + } + offset-- + } + return s[:offset+1] +} + +// Ftoa converts a float to a string with no trailing zeros. +func Ftoa(num float64) string { + return stripTrailingZeros(strconv.FormatFloat(num, 'f', 6, 64)) +} diff --git a/vendor/github.com/dustin/go-humanize/ftoa_test.go b/vendor/github.com/dustin/go-humanize/ftoa_test.go new file mode 100644 index 0000000..276d411 --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/ftoa_test.go @@ -0,0 +1,55 @@ +package humanize + +import ( + "fmt" + "regexp" + "strconv" + "testing" +) + +func TestFtoa(t *testing.T) { + testList{ + {"200", Ftoa(200), "200"}, + {"2", Ftoa(2), "2"}, + {"2.2", Ftoa(2.2), "2.2"}, + {"2.02", Ftoa(2.02), "2.02"}, + {"200.02", Ftoa(200.02), "200.02"}, + }.validate(t) +} + +func BenchmarkFtoaRegexTrailing(b *testing.B) { + trailingZerosRegex := regexp.MustCompile(`\.?0+$`) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + trailingZerosRegex.ReplaceAllString("2.00000", "") + trailingZerosRegex.ReplaceAllString("2.0000", "") + trailingZerosRegex.ReplaceAllString("2.000", "") + trailingZerosRegex.ReplaceAllString("2.00", "") + trailingZerosRegex.ReplaceAllString("2.0", "") + trailingZerosRegex.ReplaceAllString("2", "") + } +} + +func BenchmarkFtoaFunc(b *testing.B) { + for i := 0; i < b.N; i++ { + stripTrailingZeros("2.00000") + stripTrailingZeros("2.0000") + stripTrailingZeros("2.000") + stripTrailingZeros("2.00") + stripTrailingZeros("2.0") + stripTrailingZeros("2") + } +} + +func BenchmarkFmtF(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = fmt.Sprintf("%f", 2.03584) + } +} + +func BenchmarkStrconvF(b *testing.B) { + for i := 0; i < b.N; i++ { + strconv.FormatFloat(2.03584, 'f', 6, 64) + } +} diff --git a/vendor/github.com/dustin/go-humanize/humanize.go b/vendor/github.com/dustin/go-humanize/humanize.go new file mode 100644 index 0000000..a2c2da3 --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/humanize.go @@ -0,0 +1,8 @@ +/* +Package humanize converts boring ugly numbers to human-friendly strings and back. + +Durations can be turned into strings such as "3 days ago", numbers +representing sizes like 82854982 into useful strings like, "83 MB" or +"79 MiB" (whichever you prefer). +*/ +package humanize diff --git a/vendor/github.com/dustin/go-humanize/number.go b/vendor/github.com/dustin/go-humanize/number.go new file mode 100644 index 0000000..dec6186 --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/number.go @@ -0,0 +1,192 @@ +package humanize + +/* +Slightly adapted from the source to fit go-humanize. + +Author: https://github.com/gorhill +Source: https://gist.github.com/gorhill/5285193 + +*/ + +import ( + "math" + "strconv" +) + +var ( + renderFloatPrecisionMultipliers = [...]float64{ + 1, + 10, + 100, + 1000, + 10000, + 100000, + 1000000, + 10000000, + 100000000, + 1000000000, + } + + renderFloatPrecisionRounders = [...]float64{ + 0.5, + 0.05, + 0.005, + 0.0005, + 0.00005, + 0.000005, + 0.0000005, + 0.00000005, + 0.000000005, + 0.0000000005, + } +) + +// FormatFloat produces a formatted number as string based on the following user-specified criteria: +// * thousands separator +// * decimal separator +// * decimal precision +// +// Usage: s := RenderFloat(format, n) +// The format parameter tells how to render the number n. +// +// See examples: http://play.golang.org/p/LXc1Ddm1lJ +// +// Examples of format strings, given n = 12345.6789: +// "#,###.##" => "12,345.67" +// "#,###." => "12,345" +// "#,###" => "12345,678" +// "#\u202F###,##" => "12 345,68" +// "#.###,###### => 12.345,678900 +// "" (aka default format) => 12,345.67 +// +// The highest precision allowed is 9 digits after the decimal symbol. +// There is also a version for integer number, FormatInteger(), +// which is convenient for calls within template. +func FormatFloat(format string, n float64) string { + // Special cases: + // NaN = "NaN" + // +Inf = "+Infinity" + // -Inf = "-Infinity" + if math.IsNaN(n) { + return "NaN" + } + if n > math.MaxFloat64 { + return "Infinity" + } + if n < -math.MaxFloat64 { + return "-Infinity" + } + + // default format + precision := 2 + decimalStr := "." + thousandStr := "," + positiveStr := "" + negativeStr := "-" + + if len(format) > 0 { + format := []rune(format) + + // If there is an explicit format directive, + // then default values are these: + precision = 9 + thousandStr = "" + + // collect indices of meaningful formatting directives + formatIndx := []int{} + for i, char := range format { + if char != '#' && char != '0' { + formatIndx = append(formatIndx, i) + } + } + + if len(formatIndx) > 0 { + // Directive at index 0: + // Must be a '+' + // Raise an error if not the case + // index: 0123456789 + // +0.000,000 + // +000,000.0 + // +0000.00 + // +0000 + if formatIndx[0] == 0 { + if format[formatIndx[0]] != '+' { + panic("RenderFloat(): invalid positive sign directive") + } + positiveStr = "+" + formatIndx = formatIndx[1:] + } + + // Two directives: + // First is thousands separator + // Raise an error if not followed by 3-digit + // 0123456789 + // 0.000,000 + // 000,000.00 + if len(formatIndx) == 2 { + if (formatIndx[1] - formatIndx[0]) != 4 { + panic("RenderFloat(): thousands separator directive must be followed by 3 digit-specifiers") + } + thousandStr = string(format[formatIndx[0]]) + formatIndx = formatIndx[1:] + } + + // One directive: + // Directive is decimal separator + // The number of digit-specifier following the separator indicates wanted precision + // 0123456789 + // 0.00 + // 000,0000 + if len(formatIndx) == 1 { + decimalStr = string(format[formatIndx[0]]) + precision = len(format) - formatIndx[0] - 1 + } + } + } + + // generate sign part + var signStr string + if n >= 0.000000001 { + signStr = positiveStr + } else if n <= -0.000000001 { + signStr = negativeStr + n = -n + } else { + signStr = "" + n = 0.0 + } + + // split number into integer and fractional parts + intf, fracf := math.Modf(n + renderFloatPrecisionRounders[precision]) + + // generate integer part string + intStr := strconv.FormatInt(int64(intf), 10) + + // add thousand separator if required + if len(thousandStr) > 0 { + for i := len(intStr); i > 3; { + i -= 3 + intStr = intStr[:i] + thousandStr + intStr[i:] + } + } + + // no fractional part, we can leave now + if precision == 0 { + return signStr + intStr + } + + // generate fractional part + fracStr := strconv.Itoa(int(fracf * renderFloatPrecisionMultipliers[precision])) + // may need padding + if len(fracStr) < precision { + fracStr = "000000000000000"[:precision-len(fracStr)] + fracStr + } + + return signStr + intStr + decimalStr + fracStr +} + +// FormatInteger produces a formatted number as string. +// See FormatFloat. +func FormatInteger(format string, n int) string { + return FormatFloat(format, float64(n)) +} diff --git a/vendor/github.com/dustin/go-humanize/number_test.go b/vendor/github.com/dustin/go-humanize/number_test.go new file mode 100644 index 0000000..516f337 --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/number_test.go @@ -0,0 +1,79 @@ +package humanize + +import ( + "math" + "testing" +) + +type TestStruct struct { + name string + format string + num float64 + formatted string +} + +func TestFormatFloat(t *testing.T) { + tests := []TestStruct{ + {"default", "", 12345.6789, "12,345.68"}, + {"#", "#", 12345.6789, "12345.678900000"}, + {"#.", "#.", 12345.6789, "12346"}, + {"#,#", "#,#", 12345.6789, "12345,7"}, + {"#,##", "#,##", 12345.6789, "12345,68"}, + {"#,###", "#,###", 12345.6789, "12345,679"}, + {"#,###.", "#,###.", 12345.6789, "12,346"}, + {"#,###.##", "#,###.##", 12345.6789, "12,345.68"}, + {"#,###.###", "#,###.###", 12345.6789, "12,345.679"}, + {"#,###.####", "#,###.####", 12345.6789, "12,345.6789"}, + {"#.###,######", "#.###,######", 12345.6789, "12.345,678900"}, + {"bug46", "#,###.##", 52746220055.92342, "52,746,220,055.92"}, + {"#\u202f###,##", "#\u202f###,##", 12345.6789, "12 345,68"}, + + // special cases + {"NaN", "#", math.NaN(), "NaN"}, + {"+Inf", "#", math.Inf(1), "Infinity"}, + {"-Inf", "#", math.Inf(-1), "-Infinity"}, + {"signStr <= -0.000000001", "", -0.000000002, "-0.00"}, + {"signStr = 0", "", 0, "0.00"}, + {"Format directive must start with +", "+000", 12345.6789, "+12345.678900000"}, + } + + for _, test := range tests { + got := FormatFloat(test.format, test.num) + if got != test.formatted { + t.Errorf("On %v (%v, %v), got %v, wanted %v", + test.name, test.format, test.num, got, test.formatted) + } + } + // Test a single integer + got := FormatInteger("#", 12345) + if got != "12345.000000000" { + t.Errorf("On %v (%v, %v), got %v, wanted %v", + "integerTest", "#", 12345, got, "12345.000000000") + } + // Test the things that could panic + panictests := []TestStruct{ + {"RenderFloat(): invalid positive sign directive", "-", 12345.6789, "12,345.68"}, + {"RenderFloat(): thousands separator directive must be followed by 3 digit-specifiers", "0.01", 12345.6789, "12,345.68"}, + } + for _, test := range panictests { + didPanic := false + var message interface{} + func() { + + defer func() { + if message = recover(); message != nil { + didPanic = true + } + }() + + // call the target function + _ = FormatFloat(test.format, test.num) + + }() + if didPanic != true { + t.Errorf("On %v, should have panic and did not.", + test.name) + } + } + +} diff --git a/vendor/github.com/dustin/go-humanize/ordinals.go b/vendor/github.com/dustin/go-humanize/ordinals.go new file mode 100644 index 0000000..43d88a8 --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/ordinals.go @@ -0,0 +1,25 @@ +package humanize + +import "strconv" + +// Ordinal gives you the input number in a rank/ordinal format. +// +// Ordinal(3) -> 3rd +func Ordinal(x int) string { + suffix := "th" + switch x % 10 { + case 1: + if x%100 != 11 { + suffix = "st" + } + case 2: + if x%100 != 12 { + suffix = "nd" + } + case 3: + if x%100 != 13 { + suffix = "rd" + } + } + return strconv.Itoa(x) + suffix +} diff --git a/vendor/github.com/dustin/go-humanize/ordinals_test.go b/vendor/github.com/dustin/go-humanize/ordinals_test.go new file mode 100644 index 0000000..51d85ee --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/ordinals_test.go @@ -0,0 +1,22 @@ +package humanize + +import ( + "testing" +) + +func TestOrdinals(t *testing.T) { + testList{ + {"0", Ordinal(0), "0th"}, + {"1", Ordinal(1), "1st"}, + {"2", Ordinal(2), "2nd"}, + {"3", Ordinal(3), "3rd"}, + {"4", Ordinal(4), "4th"}, + {"10", Ordinal(10), "10th"}, + {"11", Ordinal(11), "11th"}, + {"12", Ordinal(12), "12th"}, + {"13", Ordinal(13), "13th"}, + {"101", Ordinal(101), "101st"}, + {"102", Ordinal(102), "102nd"}, + {"103", Ordinal(103), "103rd"}, + }.validate(t) +} diff --git a/vendor/github.com/dustin/go-humanize/si.go b/vendor/github.com/dustin/go-humanize/si.go new file mode 100644 index 0000000..b24e481 --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/si.go @@ -0,0 +1,113 @@ +package humanize + +import ( + "errors" + "math" + "regexp" + "strconv" +) + +var siPrefixTable = map[float64]string{ + -24: "y", // yocto + -21: "z", // zepto + -18: "a", // atto + -15: "f", // femto + -12: "p", // pico + -9: "n", // nano + -6: "µ", // micro + -3: "m", // milli + 0: "", + 3: "k", // kilo + 6: "M", // mega + 9: "G", // giga + 12: "T", // tera + 15: "P", // peta + 18: "E", // exa + 21: "Z", // zetta + 24: "Y", // yotta +} + +var revSIPrefixTable = revfmap(siPrefixTable) + +// revfmap reverses the map and precomputes the power multiplier +func revfmap(in map[float64]string) map[string]float64 { + rv := map[string]float64{} + for k, v := range in { + rv[v] = math.Pow(10, k) + } + return rv +} + +var riParseRegex *regexp.Regexp + +func init() { + ri := `^([\-0-9.]+)\s?([` + for _, v := range siPrefixTable { + ri += v + } + ri += `]?)(.*)` + + riParseRegex = regexp.MustCompile(ri) +} + +// ComputeSI finds the most appropriate SI prefix for the given number +// and returns the prefix along with the value adjusted to be within +// that prefix. +// +// See also: SI, ParseSI. +// +// e.g. ComputeSI(2.2345e-12) -> (2.2345, "p") +func ComputeSI(input float64) (float64, string) { + if input == 0 { + return 0, "" + } + mag := math.Abs(input) + exponent := math.Floor(logn(mag, 10)) + exponent = math.Floor(exponent/3) * 3 + + value := mag / math.Pow(10, exponent) + + // Handle special case where value is exactly 1000.0 + // Should return 1 M instead of 1000 k + if value == 1000.0 { + exponent += 3 + value = mag / math.Pow(10, exponent) + } + + value = math.Copysign(value, input) + + prefix := siPrefixTable[exponent] + return value, prefix +} + +// SI returns a string with default formatting. +// +// SI uses Ftoa to format float value, removing trailing zeros. +// +// See also: ComputeSI, ParseSI. +// +// e.g. SI(1000000, "B") -> 1 MB +// e.g. SI(2.2345e-12, "F") -> 2.2345 pF +func SI(input float64, unit string) string { + value, prefix := ComputeSI(input) + return Ftoa(value) + " " + prefix + unit +} + +var errInvalid = errors.New("invalid input") + +// ParseSI parses an SI string back into the number and unit. +// +// See also: SI, ComputeSI. +// +// e.g. ParseSI("2.2345 pF") -> (2.2345e-12, "F", nil) +func ParseSI(input string) (float64, string, error) { + found := riParseRegex.FindStringSubmatch(input) + if len(found) != 4 { + return 0, "", errInvalid + } + mag := revSIPrefixTable[found[2]] + unit := found[3] + + base, err := strconv.ParseFloat(found[1], 64) + return base * mag, unit, err +} diff --git a/vendor/github.com/dustin/go-humanize/si_test.go b/vendor/github.com/dustin/go-humanize/si_test.go new file mode 100644 index 0000000..bc5bac6 --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/si_test.go @@ -0,0 +1,101 @@ +package humanize + +import ( + "math" + "testing" +) + +func TestSI(t *testing.T) { + tests := []struct { + name string + num float64 + formatted string + }{ + {"e-24", 1e-24, "1 yF"}, + {"e-21", 1e-21, "1 zF"}, + {"e-18", 1e-18, "1 aF"}, + {"e-15", 1e-15, "1 fF"}, + {"e-12", 1e-12, "1 pF"}, + {"e-12", 2.2345e-12, "2.2345 pF"}, + {"e-12", 2.23e-12, "2.23 pF"}, + {"e-11", 2.23e-11, "22.3 pF"}, + {"e-10", 2.2e-10, "220 pF"}, + {"e-9", 2.2e-9, "2.2 nF"}, + {"e-8", 2.2e-8, "22 nF"}, + {"e-7", 2.2e-7, "220 nF"}, + {"e-6", 2.2e-6, "2.2 µF"}, + {"e-6", 1e-6, "1 µF"}, + {"e-5", 2.2e-5, "22 µF"}, + {"e-4", 2.2e-4, "220 µF"}, + {"e-3", 2.2e-3, "2.2 mF"}, + {"e-2", 2.2e-2, "22 mF"}, + {"e-1", 2.2e-1, "220 mF"}, + {"e+0", 2.2e-0, "2.2 F"}, + {"e+0", 2.2, "2.2 F"}, + {"e+1", 2.2e+1, "22 F"}, + {"0", 0, "0 F"}, + {"e+1", 22, "22 F"}, + {"e+2", 2.2e+2, "220 F"}, + {"e+2", 220, "220 F"}, + {"e+3", 2.2e+3, "2.2 kF"}, + {"e+3", 2200, "2.2 kF"}, + {"e+4", 2.2e+4, "22 kF"}, + {"e+4", 22000, "22 kF"}, + {"e+5", 2.2e+5, "220 kF"}, + {"e+6", 2.2e+6, "2.2 MF"}, + {"e+6", 1e+6, "1 MF"}, + {"e+7", 2.2e+7, "22 MF"}, + {"e+8", 2.2e+8, "220 MF"}, + {"e+9", 2.2e+9, "2.2 GF"}, + {"e+10", 2.2e+10, "22 GF"}, + {"e+11", 2.2e+11, "220 GF"}, + {"e+12", 2.2e+12, "2.2 TF"}, + {"e+15", 2.2e+15, "2.2 PF"}, + {"e+18", 2.2e+18, "2.2 EF"}, + {"e+21", 2.2e+21, "2.2 ZF"}, + {"e+24", 2.2e+24, "2.2 YF"}, + + // special case + {"1F", 1000 * 1000, "1 MF"}, + {"1F", 1e6, "1 MF"}, + + // negative number + {"-100 F", -100, "-100 F"}, + } + + for _, test := range tests { + got := SI(test.num, "F") + if got != test.formatted { + t.Errorf("On %v (%v), got %v, wanted %v", + test.name, test.num, got, test.formatted) + } + + gotf, gotu, err := ParseSI(test.formatted) + if err != nil { + t.Errorf("Error parsing %v (%v): %v", test.name, test.formatted, err) + continue + } + + if math.Abs(1-(gotf/test.num)) > 0.01 { + t.Errorf("On %v (%v), got %v, wanted %v (±%v)", + test.name, test.formatted, gotf, test.num, + math.Abs(1-(gotf/test.num))) + } + if gotu != "F" { + t.Errorf("On %v (%v), expected unit F, got %v", + test.name, test.formatted, gotu) + } + } + + // Parse error + gotf, gotu, err := ParseSI("x1.21JW") // 1.21 jigga whats + if err == nil { + t.Errorf("Expected error on x1.21JW, got %v %v", gotf, gotu) + } +} + +func BenchmarkParseSI(b *testing.B) { + for i := 0; i < b.N; i++ { + ParseSI("2.2346ZB") + } +} diff --git a/vendor/github.com/dustin/go-humanize/times.go b/vendor/github.com/dustin/go-humanize/times.go new file mode 100644 index 0000000..b311f11 --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/times.go @@ -0,0 +1,117 @@ +package humanize + +import ( + "fmt" + "math" + "sort" + "time" +) + +// Seconds-based time units +const ( + Day = 24 * time.Hour + Week = 7 * Day + Month = 30 * Day + Year = 12 * Month + LongTime = 37 * Year +) + +// Time formats a time into a relative string. +// +// Time(someT) -> "3 weeks ago" +func Time(then time.Time) string { + return RelTime(then, time.Now(), "ago", "from now") +} + +// A RelTimeMagnitude struct contains a relative time point at which +// the relative format of time will switch to a new format string. A +// slice of these in ascending order by their "D" field is passed to +// CustomRelTime to format durations. +// +// The Format field is a string that may contain a "%s" which will be +// replaced with the appropriate signed label (e.g. "ago" or "from +// now") and a "%d" that will be replaced by the quantity. +// +// The DivBy field is the amount of time the time difference must be +// divided by in order to display correctly. +// +// e.g. if D is 2*time.Minute and you want to display "%d minutes %s" +// DivBy should be time.Minute so whatever the duration is will be +// expressed in minutes. +type RelTimeMagnitude struct { + D time.Duration + Format string + DivBy time.Duration +} + +var defaultMagnitudes = []RelTimeMagnitude{ + {time.Second, "now", time.Second}, + {2 * time.Second, "1 second %s", 1}, + {time.Minute, "%d seconds %s", time.Second}, + {2 * time.Minute, "1 minute %s", 1}, + {time.Hour, "%d minutes %s", time.Minute}, + {2 * time.Hour, "1 hour %s", 1}, + {Day, "%d hours %s", time.Hour}, + {2 * Day, "1 day %s", 1}, + {Week, "%d days %s", Day}, + {2 * Week, "1 week %s", 1}, + {Month, "%d weeks %s", Week}, + {2 * Month, "1 month %s", 1}, + {Year, "%d months %s", Month}, + {18 * Month, "1 year %s", 1}, + {2 * Year, "2 years %s", 1}, + {LongTime, "%d years %s", Year}, + {math.MaxInt64, "a long while %s", 1}, +} + +// RelTime formats a time into a relative string. +// +// It takes two times and two labels. In addition to the generic time +// delta string (e.g. 5 minutes), the labels are used applied so that +// the label corresponding to the smaller time is applied. +// +// RelTime(timeInPast, timeInFuture, "earlier", "later") -> "3 weeks earlier" +func RelTime(a, b time.Time, albl, blbl string) string { + return CustomRelTime(a, b, albl, blbl, defaultMagnitudes) +} + +// CustomRelTime formats a time into a relative string. +// +// It takes two times two labels and a table of relative time formats. +// In addition to the generic time delta string (e.g. 5 minutes), the +// labels are used applied so that the label corresponding to the +// smaller time is applied. +func CustomRelTime(a, b time.Time, albl, blbl string, magnitudes []RelTimeMagnitude) string { + lbl := albl + diff := b.Sub(a) + + if a.After(b) { + lbl = blbl + diff = a.Sub(b) + } + + n := sort.Search(len(magnitudes), func(i int) bool { + return magnitudes[i].D >= diff + }) + + if n >= len(magnitudes) { + n = len(magnitudes) - 1 + } + mag := magnitudes[n] + args := []interface{}{} + escaped := false + for _, ch := range mag.Format { + if escaped { + switch ch { + case 's': + args = append(args, lbl) + case 'd': + args = append(args, diff/mag.DivBy) + } + escaped = false + } else { + escaped = ch == '%' + } + } + return fmt.Sprintf(mag.Format, args...) +} diff --git a/vendor/github.com/dustin/go-humanize/times_test.go b/vendor/github.com/dustin/go-humanize/times_test.go new file mode 100644 index 0000000..c90a71b --- /dev/null +++ b/vendor/github.com/dustin/go-humanize/times_test.go @@ -0,0 +1,113 @@ +package humanize + +import ( + "math" + "testing" + "time" +) + +func TestPast(t *testing.T) { + now := time.Now() + testList{ + {"now", Time(now), "now"}, + {"1 second ago", Time(now.Add(-1 * time.Second)), "1 second ago"}, + {"12 seconds ago", Time(now.Add(-12 * time.Second)), "12 seconds ago"}, + {"30 seconds ago", Time(now.Add(-30 * time.Second)), "30 seconds ago"}, + {"45 seconds ago", Time(now.Add(-45 * time.Second)), "45 seconds ago"}, + {"1 minute ago", Time(now.Add(-63 * time.Second)), "1 minute ago"}, + {"15 minutes ago", Time(now.Add(-15 * time.Minute)), "15 minutes ago"}, + {"1 hour ago", Time(now.Add(-63 * time.Minute)), "1 hour ago"}, + {"2 hours ago", Time(now.Add(-2 * time.Hour)), "2 hours ago"}, + {"21 hours ago", Time(now.Add(-21 * time.Hour)), "21 hours ago"}, + {"1 day ago", Time(now.Add(-26 * time.Hour)), "1 day ago"}, + {"2 days ago", Time(now.Add(-49 * time.Hour)), "2 days ago"}, + {"3 days ago", Time(now.Add(-3 * Day)), "3 days ago"}, + {"1 week ago (1)", Time(now.Add(-7 * Day)), "1 week ago"}, + {"1 week ago (2)", Time(now.Add(-12 * Day)), "1 week ago"}, + {"2 weeks ago", Time(now.Add(-15 * Day)), "2 weeks ago"}, + {"1 month ago", Time(now.Add(-39 * Day)), "1 month ago"}, + {"3 months ago", Time(now.Add(-99 * Day)), "3 months ago"}, + {"1 year ago (1)", Time(now.Add(-365 * Day)), "1 year ago"}, + {"1 year ago (1)", Time(now.Add(-400 * Day)), "1 year ago"}, + {"2 years ago (1)", Time(now.Add(-548 * Day)), "2 years ago"}, + {"2 years ago (2)", Time(now.Add(-725 * Day)), "2 years ago"}, + {"2 years ago (3)", Time(now.Add(-800 * Day)), "2 years ago"}, + {"3 years ago", Time(now.Add(-3 * Year)), "3 years ago"}, + {"long ago", Time(now.Add(-LongTime)), "a long while ago"}, + }.validate(t) +} + +func TestFuture(t *testing.T) { + // Add a little time so that these things properly line up in + // the future. + now := time.Now().Add(time.Millisecond * 250) + testList{ + {"now", Time(now), "now"}, + {"1 second from now", Time(now.Add(+1 * time.Second)), "1 second from now"}, + {"12 seconds from now", Time(now.Add(+12 * time.Second)), "12 seconds from now"}, + {"30 seconds from now", Time(now.Add(+30 * time.Second)), "30 seconds from now"}, + {"45 seconds from now", Time(now.Add(+45 * time.Second)), "45 seconds from now"}, + {"15 minutes from now", Time(now.Add(+15 * time.Minute)), "15 minutes from now"}, + {"2 hours from now", Time(now.Add(+2 * time.Hour)), "2 hours from now"}, + {"21 hours from now", Time(now.Add(+21 * time.Hour)), "21 hours from now"}, + {"1 day from now", Time(now.Add(+26 * time.Hour)), "1 day from now"}, + {"2 days from now", Time(now.Add(+49 * time.Hour)), "2 days from now"}, + {"3 days from now", Time(now.Add(+3 * Day)), "3 days from now"}, + {"1 week from now (1)", Time(now.Add(+7 * Day)), "1 week from now"}, + {"1 week from now (2)", Time(now.Add(+12 * Day)), "1 week from now"}, + {"2 weeks from now", Time(now.Add(+15 * Day)), "2 weeks from now"}, + {"1 month from now", Time(now.Add(+30 * Day)), "1 month from now"}, + {"1 year from now", Time(now.Add(+365 * Day)), "1 year from now"}, + {"2 years from now", Time(now.Add(+2 * Year)), "2 years from now"}, + {"a while from now", Time(now.Add(+LongTime)), "a long while from now"}, + }.validate(t) +} + +func TestRange(t *testing.T) { + start := time.Time{} + end := time.Unix(math.MaxInt64, math.MaxInt64) + x := RelTime(start, end, "ago", "from now") + if x != "a long while from now" { + t.Errorf("Expected a long while from now, got %q", x) + } +} + +func TestCustomRelTime(t *testing.T) { + now := time.Now().Add(time.Millisecond * 250) + magnitudes := []RelTimeMagnitude{ + {time.Second, "now", time.Second}, + {2 * time.Second, "1 second %s", 1}, + {time.Minute, "%d seconds %s", time.Second}, + {Day - time.Second, "%d minutes %s", time.Minute}, + {Day, "%d hours %s", time.Hour}, + {2 * Day, "1 day %s", 1}, + {Week, "%d days %s", Day}, + {2 * Week, "1 week %s", 1}, + {6 * Month, "%d weeks %s", Week}, + {Year, "%d months %s", Month}, + } + customRelTime := func(then time.Time) string { + return CustomRelTime(then, time.Now(), "ago", "from now", magnitudes) + } + testList{ + {"now", customRelTime(now), "now"}, + {"1 second from now", customRelTime(now.Add(+1 * time.Second)), "1 second from now"}, + {"12 seconds from now", customRelTime(now.Add(+12 * time.Second)), "12 seconds from now"}, + {"30 seconds from now", customRelTime(now.Add(+30 * time.Second)), "30 seconds from now"}, + {"45 seconds from now", customRelTime(now.Add(+45 * time.Second)), "45 seconds from now"}, + {"15 minutes from now", customRelTime(now.Add(+15 * time.Minute)), "15 minutes from now"}, + {"2 hours from now", customRelTime(now.Add(+2 * time.Hour)), "120 minutes from now"}, + {"21 hours from now", customRelTime(now.Add(+21 * time.Hour)), "1260 minutes from now"}, + {"1 day from now", customRelTime(now.Add(+26 * time.Hour)), "1 day from now"}, + {"2 days from now", customRelTime(now.Add(+49 * time.Hour)), "2 days from now"}, + {"3 days from now", customRelTime(now.Add(+3 * Day)), "3 days from now"}, + {"1 week from now (1)", customRelTime(now.Add(+7 * Day)), "1 week from now"}, + {"1 week from now (2)", customRelTime(now.Add(+12 * Day)), "1 week from now"}, + {"2 weeks from now", customRelTime(now.Add(+15 * Day)), "2 weeks from now"}, + {"1 month from now", customRelTime(now.Add(+30 * Day)), "4 weeks from now"}, + {"6 months from now", customRelTime(now.Add(+6*Month - time.Second)), "25 weeks from now"}, + {"1 year from now", customRelTime(now.Add(+365 * Day)), "12 months from now"}, + {"2 years from now", customRelTime(now.Add(+2 * Year)), "24 months from now"}, + {"a while from now", customRelTime(now.Add(+LongTime)), "444 months from now"}, + }.validate(t) +} diff --git a/vendor/github.com/fluffle/goirc/.gitignore b/vendor/github.com/fluffle/goirc/.gitignore new file mode 100644 index 0000000..1cbcab5 --- /dev/null +++ b/vendor/github.com/fluffle/goirc/.gitignore @@ -0,0 +1,8 @@ +/gobot +*.[568] +_obj/ +_test/ +*.swp +*~ +*.out +/.gitconfig diff --git a/vendor/github.com/fluffle/goirc/.travis.yml b/vendor/github.com/fluffle/goirc/.travis.yml new file mode 100644 index 0000000..420a9a8 --- /dev/null +++ b/vendor/github.com/fluffle/goirc/.travis.yml @@ -0,0 +1,19 @@ +--- +language: go + +go: + - 1.5 + - 1.5.3 + - 1.6 + +sudo : false + +notifications: + irc: + channels: + - "irc.pl0rt.org#sp0rklf" + skip_join: true + +script: + - if [ "$TRAVIS_REPO_SLUG" != "fluffle/goirc" ] ; then ln -s "$HOME/gopath/src/github.com/$TRAVIS_REPO_SLUG" /home/travis/gopath/src/github.com/fluffle/goirc ; fi + - go test -v ./... diff --git a/vendor/github.com/fluffle/goirc/LICENSE b/vendor/github.com/fluffle/goirc/LICENSE new file mode 100644 index 0000000..e1b4d5c --- /dev/null +++ b/vendor/github.com/fluffle/goirc/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009+ Alex Bramley. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/fluffle/goirc/README.md b/vendor/github.com/fluffle/goirc/README.md new file mode 100644 index 0000000..2c12e84 --- /dev/null +++ b/vendor/github.com/fluffle/goirc/README.md @@ -0,0 +1,119 @@ +[![Build Status](https://api.travis-ci.org/fluffle/goirc.svg)](https://travis-ci.org/fluffle/goirc) + +GoIRC Client Framework +====================== + +### Acquiring and Building + +Pretty simple, really: + + go get github.com/fluffle/goirc/client + +There is some example code that demonstrates usage of the library in `client.go`. This will connect to freenode and join `#go-nuts` by default, so be careful ;-) + +See `fix/goirc.go` and the README there for a quick way to migrate from the +old `go1` API. + +### Using the framework + +Synopsis: +```go +package main + +import ( + "crypto/tls" + "fmt" + + irc "github.com/fluffle/goirc/client" +) + +func main() { + // Creating a simple IRC client is simple. + c := irc.SimpleClient("nick") + + // Or, create a config and fiddle with it first: + cfg := irc.NewConfig("nick") + cfg.SSL = true + cfg.SSLConfig = &tls.Config{ServerName: "irc.freenode.net"} + cfg.Server = "irc.freenode.net:7000" + cfg.NewNick = func(n string) string { return n + "^" } + c = irc.Client(cfg) + + // Add handlers to do things here! + // e.g. join a channel on connect. + c.HandleFunc(irc.CONNECTED, + func(conn *irc.Conn, line *irc.Line) { conn.Join("#channel") }) + // And a signal on disconnect + quit := make(chan bool) + c.HandleFunc(irc.DISCONNECTED, + func(conn *irc.Conn, line *irc.Line) { quit <- true }) + + // Tell client to connect. + if err := c.Connect(); err != nil { + fmt.Printf("Connection error: %s\n", err.Error()) + } + + // With a "simple" client, set Server before calling Connect... + c.Config().Server = "irc.freenode.net" + + // ... or, use ConnectTo instead. + if err := c.ConnectTo("irc.freenode.net"); err != nil { + fmt.Printf("Connection error: %s\n", err.Error()) + } + + // Wait for disconnect + <-quit +} +``` + +The test client provides a good (if basic) example of how to use the framework. +Reading `client/handlers.go` gives a more in-depth look at how handlers can be +written. Commands to be sent to the server (e.g. PRIVMSG) are methods of the +main `*Conn` struct, and can be found in `client/commands.go` (not all of the +possible IRC commands are implemented yet). Events are produced directly from +the messages from the IRC server, so you have to handle e.g. "332" for +`RPL_TOPIC` to get the topic for a channel. + +The vast majority of handlers implemented within the framework deal with state +tracking of all nicks in any channels that the client is also present in. These +handlers are in `client/state_handlers.go`. State tracking is optional, disabled +by default, and can be enabled and disabled by calling `EnableStateTracking()` +and `DisableStateTracking()` respectively. Doing this while connected to an IRC +server will probably result in an inconsistent state and a lot of warnings to +STDERR ;-) + +### Misc. + +Sorry the documentation is crap. Use the source, Luke. + +[Feedback](mailto:a.bramley@gmail.com) on design decisions is welcome. I am +indebted to Matt Gruen for his work on +[go-bot](http://code.google.com/p/go-bot/source/browse/irc.go) which inspired +the re-organisation and channel-based communication structure of `*Conn.send()` +and `*Conn.recv()`. I'm sure things could be more asynchronous, still. + +This code is (c) 2009-15 Alex Bramley, and released under the same licence terms +as Go itself. + +Contributions gratefully received from: + + - [3onyc](https://github.com/3onyc) + - [bramp](https://github.com/bramp) + - [cgt](https://github.com/cgt) + - [iopred](https://github.com/iopred) + - [Krayons](https://github.com/Krayons) + - [StalkR](https://github.com/StalkR) + - [sztanpet](https://github.com/sztanpet) + - [wathiede](https://github.com/wathiede) + - [scrapbird](https://github.com/scrapbird) + - [soul9](https://github.com/soul9) + - [jakebailey](https://github.com/jakebailey) + - [stapelberg](https://github.com/stapelberg) + +And thanks to the following for minor doc/fix PRs: + + - [tmcarr](https://github.com/tmcarr) + - [Gentux](https://github.com/Gentux) + - [kidanger](https://github.com/kidanger) + - [ripcurld00d](https://github.com/ripcurld00d) + - [bob-smith](https://github.com/bob-smith) diff --git a/vendor/github.com/fluffle/goirc/client.go b/vendor/github.com/fluffle/goirc/client.go new file mode 100644 index 0000000..b71d706 --- /dev/null +++ b/vendor/github.com/fluffle/goirc/client.go @@ -0,0 +1,104 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "os" + "strings" + + irc "github.com/fluffle/goirc/client" + "github.com/fluffle/goirc/logging/glog" +) + +var host *string = flag.String("host", "irc.freenode.net", "IRC server") +var channel *string = flag.String("channel", "#go-nuts", "IRC channel") + +func main() { + flag.Parse() + glog.Init() + + // create new IRC connection + c := irc.SimpleClient("GoTest", "gotest") + c.EnableStateTracking() + c.HandleFunc("connected", + func(conn *irc.Conn, line *irc.Line) { conn.Join(*channel) }) + + // Set up a handler to notify of disconnect events. + quit := make(chan bool) + c.HandleFunc("disconnected", + func(conn *irc.Conn, line *irc.Line) { quit <- true }) + + // set up a goroutine to read commands from stdin + in := make(chan string, 4) + reallyquit := false + go func() { + con := bufio.NewReader(os.Stdin) + for { + s, err := con.ReadString('\n') + if err != nil { + // wha?, maybe ctrl-D... + close(in) + break + } + // no point in sending empty lines down the channel + if len(s) > 2 { + in <- s[0 : len(s)-1] + } + } + }() + + // set up a goroutine to do parsey things with the stuff from stdin + go func() { + for cmd := range in { + if cmd[0] == ':' { + switch idx := strings.Index(cmd, " "); { + case cmd[1] == 'd': + fmt.Printf(c.String()) + case cmd[1] == 'n': + parts := strings.Split(cmd, " ") + username := strings.TrimSpace(parts[1]) + channelname := strings.TrimSpace(parts[2]) + _, userIsOn := c.StateTracker().IsOn(channelname, username) + fmt.Printf("Checking if %s is in %s Online: %t\n", username, channelname, userIsOn) + case cmd[1] == 'f': + if len(cmd) > 2 && cmd[2] == 'e' { + // enable flooding + c.Config().Flood = true + } else if len(cmd) > 2 && cmd[2] == 'd' { + // disable flooding + c.Config().Flood = false + } + for i := 0; i < 20; i++ { + c.Privmsg("#", "flood test!") + } + case idx == -1: + continue + case cmd[1] == 'q': + reallyquit = true + c.Quit(cmd[idx+1 : len(cmd)]) + case cmd[1] == 's': + reallyquit = true + c.Close() + case cmd[1] == 'j': + c.Join(cmd[idx+1 : len(cmd)]) + case cmd[1] == 'p': + c.Part(cmd[idx+1 : len(cmd)]) + } + } else { + c.Raw(cmd) + } + } + }() + + for !reallyquit { + // connect to server + if err := c.ConnectTo(*host); err != nil { + fmt.Printf("Connection error: %s\n", err) + return + } + + // wait on quit channel + <-quit + } +} diff --git a/vendor/github.com/fluffle/goirc/client/commands.go b/vendor/github.com/fluffle/goirc/client/commands.go new file mode 100644 index 0000000..101c7d3 --- /dev/null +++ b/vendor/github.com/fluffle/goirc/client/commands.go @@ -0,0 +1,304 @@ +package client + +import ( + "fmt" + "strings" +) + +const ( + REGISTER = "REGISTER" + CONNECTED = "CONNECTED" + DISCONNECTED = "DISCONNECTED" + ACTION = "ACTION" + AWAY = "AWAY" + CAP = "CAP" + CTCP = "CTCP" + CTCPREPLY = "CTCPREPLY" + ERROR = "ERROR" + INVITE = "INVITE" + JOIN = "JOIN" + KICK = "KICK" + MODE = "MODE" + NICK = "NICK" + NOTICE = "NOTICE" + OPER = "OPER" + PART = "PART" + PASS = "PASS" + PING = "PING" + PONG = "PONG" + PRIVMSG = "PRIVMSG" + QUIT = "QUIT" + TOPIC = "TOPIC" + USER = "USER" + VERSION = "VERSION" + VHOST = "VHOST" + WHO = "WHO" + WHOIS = "WHOIS" + defaultSplit = 450 +) + +// cutNewLines() pares down a string to the part before the first "\r" or "\n". +func cutNewLines(s string) string { + r := strings.SplitN(s, "\r", 2) + r = strings.SplitN(r[0], "\n", 2) + return r[0] +} + +// indexFragment looks for the last sentence split-point (defined as one of +// the punctuation characters .:;,!?"' followed by a space) in the string s +// and returns the index in the string after that split-point. If no split- +// point is found it returns the index after the last space in s, or -1. +func indexFragment(s string) int { + max := -1 + for _, sep := range []string{". ", ": ", "; ", ", ", "! ", "? ", "\" ", "' "} { + if idx := strings.LastIndex(s, sep); idx > max { + max = idx + } + } + if max > 0 { + return max + 2 + } + if idx := strings.LastIndex(s, " "); idx > 0 { + return idx + 1 + } + return -1 +} + +// splitMessage splits a message > splitLen chars at: +// 1. the end of the last sentence fragment before splitLen +// 2. the end of the last word before splitLen +// 3. splitLen itself +func splitMessage(msg string, splitLen int) (msgs []string) { + // This is quite short ;-) + if splitLen < 13 { + splitLen = defaultSplit + } + for len(msg) > splitLen { + idx := indexFragment(msg[:splitLen-3]) + if idx < 0 { + idx = splitLen - 3 + } + msgs = append(msgs, msg[:idx]+"...") + msg = msg[idx:] + } + return append(msgs, msg) +} + +// Raw sends a raw line to the server, should really only be used for +// debugging purposes but may well come in handy. +func (conn *Conn) Raw(rawline string) { + // Avoid command injection by enforcing one command per line. + conn.out <- cutNewLines(rawline) +} + +// Pass sends a PASS command to the server. +// PASS password +func (conn *Conn) Pass(password string) { conn.Raw(PASS + " " + password) } + +// Nick sends a NICK command to the server. +// NICK nick +func (conn *Conn) Nick(nick string) { conn.Raw(NICK + " " + nick) } + +// User sends a USER command to the server. +// USER ident 12 * :name +func (conn *Conn) User(ident, name string) { + conn.Raw(USER + " " + ident + " 12 * :" + name) +} + +// Join sends a JOIN command to the server with an optional key. +// JOIN channel [key] +func (conn *Conn) Join(channel string, key ...string) { + k := "" + if len(key) > 0 { + k = " " + key[0] + } + conn.Raw(JOIN + " " + channel + k) +} + +// Part sends a PART command to the server with an optional part message. +// PART channel [:message] +func (conn *Conn) Part(channel string, message ...string) { + msg := strings.Join(message, " ") + if msg != "" { + msg = " :" + msg + } + conn.Raw(PART + " " + channel + msg) +} + +// Kick sends a KICK command to remove a nick from a channel. +// KICK channel nick [:message] +func (conn *Conn) Kick(channel, nick string, message ...string) { + msg := strings.Join(message, " ") + if msg != "" { + msg = " :" + msg + } + conn.Raw(KICK + " " + channel + " " + nick + msg) +} + +// Quit sends a QUIT command to the server with an optional quit message. +// QUIT [:message] +func (conn *Conn) Quit(message ...string) { + msg := strings.Join(message, " ") + if msg == "" { + msg = conn.cfg.QuitMessage + } + conn.Raw(QUIT + " :" + msg) +} + +// Whois sends a WHOIS command to the server. +// WHOIS nick +func (conn *Conn) Whois(nick string) { conn.Raw(WHOIS + " " + nick) } + +// Who sends a WHO command to the server. +// WHO nick +func (conn *Conn) Who(nick string) { conn.Raw(WHO + " " + nick) } + +// Privmsg sends a PRIVMSG to the target nick or channel t. +// If msg is longer than Config.SplitLen characters, multiple PRIVMSGs +// will be sent to the target containing sequential parts of msg. +// PRIVMSG t :msg +func (conn *Conn) Privmsg(t, msg string) { + prefix := PRIVMSG + " " + t + " :" + for _, s := range splitMessage(msg, conn.cfg.SplitLen) { + conn.Raw(prefix + s) + } +} + +// Privmsgln is the variadic version of Privmsg that formats the message +// that is sent to the target nick or channel t using the +// fmt.Sprintln function. +// Note: Privmsgln doesn't add the '\n' character at the end of the message. +func (conn *Conn) Privmsgln(t string, a ...interface{}) { + msg := fmt.Sprintln(a...) + // trimming the new-line character added by the fmt.Sprintln function, + // since it's irrelevant. + msg = msg[:len(msg)-1] + conn.Privmsg(t, msg) +} + +// Privmsgf is the variadic version of Privmsg that formats the message +// that is sent to the target nick or channel t using the +// fmt.Sprintf function. +func (conn *Conn) Privmsgf(t, format string, a ...interface{}) { + msg := fmt.Sprintf(format, a...) + conn.Privmsg(t, msg) +} + +// Notice sends a NOTICE to the target nick or channel t. +// If msg is longer than Config.SplitLen characters, multiple NOTICEs +// will be sent to the target containing sequential parts of msg. +// NOTICE t :msg +func (conn *Conn) Notice(t, msg string) { + for _, s := range splitMessage(msg, conn.cfg.SplitLen) { + conn.Raw(NOTICE + " " + t + " :" + s) + } +} + +// Ctcp sends a (generic) CTCP message to the target nick +// or channel t, with an optional argument. +// PRIVMSG t :\001CTCP arg\001 +func (conn *Conn) Ctcp(t, ctcp string, arg ...string) { + // We need to split again here to ensure + for _, s := range splitMessage(strings.Join(arg, " "), conn.cfg.SplitLen) { + if s != "" { + s = " " + s + } + // Using Raw rather than PRIVMSG here to avoid double-split problems. + conn.Raw(PRIVMSG + " " + t + " :\001" + strings.ToUpper(ctcp) + s + "\001") + } +} + +// CtcpReply sends a (generic) CTCP reply to the target nick +// or channel t, with an optional argument. +// NOTICE t :\001CTCP arg\001 +func (conn *Conn) CtcpReply(t, ctcp string, arg ...string) { + for _, s := range splitMessage(strings.Join(arg, " "), conn.cfg.SplitLen) { + if s != "" { + s = " " + s + } + // Using Raw rather than NOTICE here to avoid double-split problems. + conn.Raw(NOTICE + " " + t + " :\001" + strings.ToUpper(ctcp) + s + "\001") + } +} + +// Version sends a CTCP "VERSION" to the target nick or channel t. +func (conn *Conn) Version(t string) { conn.Ctcp(t, VERSION) } + +// Action sends a CTCP "ACTION" to the target nick or channel t. +func (conn *Conn) Action(t, msg string) { conn.Ctcp(t, ACTION, msg) } + +// Topic() sends a TOPIC command for a channel. +// If no topic is provided this requests that a 332 response is sent by the +// server for that channel, which can then be handled to retrieve the current +// channel topic. If a topic is provided the channel's topic will be set. +// TOPIC channel +// TOPIC channel :topic +func (conn *Conn) Topic(channel string, topic ...string) { + t := strings.Join(topic, " ") + if t != "" { + t = " :" + t + } + conn.Raw(TOPIC + " " + channel + t) +} + +// Mode sends a MODE command for a target nick or channel t. +// If no mode strings are provided this requests that a 324 response is sent +// by the server for the target. Otherwise the mode strings are concatenated +// with spaces and sent to the server. This allows e.g. +// conn.Mode("#channel", "+nsk", "mykey") +// +// MODE t +// MODE t modestring +func (conn *Conn) Mode(t string, modestring ...string) { + mode := strings.Join(modestring, " ") + if mode != "" { + mode = " " + mode + } + conn.Raw(MODE + " " + t + mode) +} + +// Away sends an AWAY command to the server. +// If a message is provided it sets the client's away status with that message, +// otherwise it resets the client's away status. +// AWAY +// AWAY :message +func (conn *Conn) Away(message ...string) { + msg := strings.Join(message, " ") + if msg != "" { + msg = " :" + msg + } + conn.Raw(AWAY + msg) +} + +// Invite sends an INVITE command to the server. +// INVITE nick channel +func (conn *Conn) Invite(nick, channel string) { + conn.Raw(INVITE + " " + nick + " " + channel) +} + +// Oper sends an OPER command to the server. +// OPER user pass +func (conn *Conn) Oper(user, pass string) { conn.Raw(OPER + " " + user + " " + pass) } + +// VHost sends a VHOST command to the server. +// VHOST user pass +func (conn *Conn) VHost(user, pass string) { conn.Raw(VHOST + " " + user + " " + pass) } + +// Ping sends a PING command to the server, which should PONG. +// PING :message +func (conn *Conn) Ping(message string) { conn.Raw(PING + " :" + message) } + +// Pong sends a PONG command to the server. +// PONG :message +func (conn *Conn) Pong(message string) { conn.Raw(PONG + " :" + message) } + +// Cap sends a CAP command to the server. +// CAP subcommand +// CAP subcommand :message +func (conn *Conn) Cap(subcommmand string, capabilities ...string) { + if len(capabilities) == 0 { + conn.Raw(CAP + " " + subcommmand) + } else { + conn.Raw(CAP + " " + subcommmand + " :" + strings.Join(capabilities, " ")) + } +} diff --git a/vendor/github.com/fluffle/goirc/client/commands_test.go b/vendor/github.com/fluffle/goirc/client/commands_test.go new file mode 100644 index 0000000..15a8a05 --- /dev/null +++ b/vendor/github.com/fluffle/goirc/client/commands_test.go @@ -0,0 +1,205 @@ +package client + +import ( + "reflect" + "testing" +) + +func TestCutNewLines(t *testing.T) { + tests := []struct{ in, out string }{ + {"", ""}, + {"foo bar", "foo bar"}, + {"foo bar\rbaz", "foo bar"}, + {"foo bar\nbaz", "foo bar"}, + {"blorp\r\n\r\nbloop", "blorp"}, + {"\n\rblaap", ""}, + {"\r\n", ""}, + {"boo\\r\\n\\n\r", "boo\\r\\n\\n"}, + } + for i, test := range tests { + out := cutNewLines(test.in) + if test.out != out { + t.Errorf("test %d: expected %q, got %q", i, test.out, out) + } + } +} + +func TestIndexFragment(t *testing.T) { + tests := []struct { + in string + out int + }{ + {"", -1}, + {"foobarbaz", -1}, + {"foo bar baz", 8}, + {"foo. bar baz", 5}, + {"foo: bar baz", 5}, + {"foo; bar baz", 5}, + {"foo, bar baz", 5}, + {"foo! bar baz", 5}, + {"foo? bar baz", 5}, + {"foo\" bar baz", 5}, + {"foo' bar baz", 5}, + {"foo. bar. baz beep", 10}, + {"foo. bar, baz beep", 10}, + } + for i, test := range tests { + out := indexFragment(test.in) + if test.out != out { + t.Errorf("test %d: expected %d, got %d", i, test.out, out) + } + } +} + +func TestSplitMessage(t *testing.T) { + tests := []struct { + in string + sp int + out []string + }{ + {"", 0, []string{""}}, + {"foo", 0, []string{"foo"}}, + {"foo bar baz beep", 0, []string{"foo bar baz beep"}}, + {"foo bar baz beep", 15, []string{"foo bar baz ...", "beep"}}, + {"foo bar, baz beep", 15, []string{"foo bar, ...", "baz beep"}}, + {"0123456789012345", 0, []string{"0123456789012345"}}, + {"0123456789012345", 15, []string{"012345678901...", "2345"}}, + {"0123456789012345", 16, []string{"0123456789012345"}}, + } + for i, test := range tests { + out := splitMessage(test.in, test.sp) + if !reflect.DeepEqual(test.out, out) { + t.Errorf("test %d: expected %q, got %q", i, test.out, out) + } + } +} + +func TestClientCommands(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + // Avoid having to type ridiculously long lines to test that + // messages longer than SplitLen are correctly sent to the server. + c.cfg.SplitLen = 23 + + c.Pass("password") + s.nc.Expect("PASS password") + + c.Nick("test") + s.nc.Expect("NICK test") + + c.User("test", "Testing IRC") + s.nc.Expect("USER test 12 * :Testing IRC") + + c.Raw("JUST a raw :line") + s.nc.Expect("JUST a raw :line") + + c.Join("#foo") + s.nc.Expect("JOIN #foo") + c.Join("#foo bar") + s.nc.Expect("JOIN #foo bar") + + c.Part("#foo") + s.nc.Expect("PART #foo") + c.Part("#foo", "Screw you guys...") + s.nc.Expect("PART #foo :Screw you guys...") + + c.Quit() + s.nc.Expect("QUIT :GoBye!") + c.Quit("I'm going home.") + s.nc.Expect("QUIT :I'm going home.") + + c.Whois("somebody") + s.nc.Expect("WHOIS somebody") + + c.Who("*@some.host.com") + s.nc.Expect("WHO *@some.host.com") + + c.Privmsg("#foo", "bar") + s.nc.Expect("PRIVMSG #foo :bar") + + c.Privmsgln("#foo", "bar") + s.nc.Expect("PRIVMSG #foo :bar") + + c.Privmsgf("#foo", "say %s", "foo") + s.nc.Expect("PRIVMSG #foo :say foo") + + c.Privmsgln("#foo", "bar", 1, 3.54, []int{24, 36}) + s.nc.Expect("PRIVMSG #foo :bar 1 3.54 [24 36]") + + c.Privmsgf("#foo", "user %d is at %s", 2, "home") + s.nc.Expect("PRIVMSG #foo :user 2 is at home") + + // 0123456789012345678901234567890123 + c.Privmsg("#foo", "foo bar baz blorp. woo woobly woo.") + s.nc.Expect("PRIVMSG #foo :foo bar baz blorp. ...") + s.nc.Expect("PRIVMSG #foo :woo woobly woo.") + + c.Privmsgln("#foo", "foo bar baz blorp. woo woobly woo.") + s.nc.Expect("PRIVMSG #foo :foo bar baz blorp. ...") + s.nc.Expect("PRIVMSG #foo :woo woobly woo.") + + c.Privmsgf("#foo", "%s %s", "foo bar baz blorp.", "woo woobly woo.") + s.nc.Expect("PRIVMSG #foo :foo bar baz blorp. ...") + s.nc.Expect("PRIVMSG #foo :woo woobly woo.") + + c.Privmsgln("#foo", "foo bar", 3.54, "blorp.", "woo", "woobly", []int{1, 2}) + s.nc.Expect("PRIVMSG #foo :foo bar 3.54 blorp. ...") + s.nc.Expect("PRIVMSG #foo :woo woobly [1 2]") + + c.Privmsgf("#foo", "%s %.2f %s %s %s %v", "foo bar", 3.54, "blorp.", "woo", "woobly", []int{1, 2}) + s.nc.Expect("PRIVMSG #foo :foo bar 3.54 blorp. ...") + s.nc.Expect("PRIVMSG #foo :woo woobly [1 2]") + + c.Notice("somebody", "something") + s.nc.Expect("NOTICE somebody :something") + + // 01234567890123456789012345678901234567 + c.Notice("somebody", "something much much longer that splits") + s.nc.Expect("NOTICE somebody :something much much ...") + s.nc.Expect("NOTICE somebody :longer that splits") + + c.Ctcp("somebody", "ping", "123456789") + s.nc.Expect("PRIVMSG somebody :\001PING 123456789\001") + + c.Ctcp("somebody", "ping", "123456789012345678901234567890") + s.nc.Expect("PRIVMSG somebody :\001PING 12345678901234567890...\001") + s.nc.Expect("PRIVMSG somebody :\001PING 1234567890\001") + + c.CtcpReply("somebody", "pong", "123456789012345678901234567890") + s.nc.Expect("NOTICE somebody :\001PONG 12345678901234567890...\001") + s.nc.Expect("NOTICE somebody :\001PONG 1234567890\001") + + c.CtcpReply("somebody", "pong", "123456789") + s.nc.Expect("NOTICE somebody :\001PONG 123456789\001") + + c.Version("somebody") + s.nc.Expect("PRIVMSG somebody :\001VERSION\001") + + c.Action("#foo", "pokes somebody") + s.nc.Expect("PRIVMSG #foo :\001ACTION pokes somebody\001") + + c.Topic("#foo") + s.nc.Expect("TOPIC #foo") + c.Topic("#foo", "la la la") + s.nc.Expect("TOPIC #foo :la la la") + + c.Mode("#foo") + s.nc.Expect("MODE #foo") + c.Mode("#foo", "+o somebody") + s.nc.Expect("MODE #foo +o somebody") + + c.Away() + s.nc.Expect("AWAY") + c.Away("Dave's not here, man.") + s.nc.Expect("AWAY :Dave's not here, man.") + + c.Invite("somebody", "#foo") + s.nc.Expect("INVITE somebody #foo") + + c.Oper("user", "pass") + s.nc.Expect("OPER user pass") + + c.VHost("user", "pass") + s.nc.Expect("VHOST user pass") +} diff --git a/vendor/github.com/fluffle/goirc/client/connection.go b/vendor/github.com/fluffle/goirc/client/connection.go new file mode 100644 index 0000000..f84621a --- /dev/null +++ b/vendor/github.com/fluffle/goirc/client/connection.go @@ -0,0 +1,581 @@ +package client + +import ( + "bufio" + "crypto/tls" + "fmt" + "io" + "net" + "net/url" + "strings" + "sync" + "time" + + "github.com/fluffle/goirc/logging" + "github.com/fluffle/goirc/state" + "golang.org/x/net/proxy" +) + +// Conn encapsulates a connection to a single IRC server. Create +// one with Client or SimpleClient. +type Conn struct { + // For preventing races on (dis)connect. + mu sync.RWMutex + + // Contains parameters that people can tweak to change client behaviour. + cfg *Config + + // Handlers + intHandlers *hSet + fgHandlers *hSet + bgHandlers *hSet + + // State tracker for nicks and channels + st state.Tracker + stRemovers []Remover + + // I/O stuff to server + dialer *net.Dialer + proxyDialer proxy.Dialer + sock net.Conn + io *bufio.ReadWriter + in chan *Line + out chan string + connected bool + + // Control channel and WaitGroup for goroutines + die chan struct{} + wg sync.WaitGroup + + // Internal counters for flood protection + badness time.Duration + lastsent time.Time +} + +// Config contains options that can be passed to Client to change the +// behaviour of the library during use. It is recommended that NewConfig +// is used to create this struct rather than instantiating one directly. +// Passing a Config with no Nick in the Me field to Client will result +// in unflattering consequences. +type Config struct { + // Set this to provide the Nick, Ident and Name for the client to use. + // It is recommended to call Conn.Me to get up-to-date information + // about the current state of the client's IRC nick after connecting. + Me *state.Nick + + // Hostname to connect to and optional connect password. + // Changing these after connection will have no effect until the + // client reconnects. + Server, Pass string + + // Are we connecting via SSL? Do we care about certificate validity? + // Changing these after connection will have no effect until the + // client reconnects. + SSL bool + SSLConfig *tls.Config + + // To connect via proxy set the proxy url here. + // Changing these after connection will have no effect until the + // client reconnects. + Proxy string + + // Local address to bind to when connecting to the server. + LocalAddr string + + // Replaceable function to customise the 433 handler's new nick. + // By default an underscore "_" is appended to the current nick. + NewNick func(string) string + + // Client->server ping frequency, in seconds. Defaults to 3m. + // Set to 0 to disable client-side pings. + PingFreq time.Duration + + // The duration before a connection timeout is triggered. Defaults to 1m. + // Set to 0 to wait indefinitely. + Timeout time.Duration + + // Set this to true to disable flood protection and false to re-enable. + Flood bool + + // Sent as the reply to a CTCP VERSION message. + Version string + + // Sent as the default QUIT message if Quit is called with no args. + QuitMessage string + + // Configurable panic recovery for all handlers. + // Defaults to logging an error, see LogPanic. + Recover func(*Conn, *Line) + + // Split PRIVMSGs, NOTICEs and CTCPs longer than SplitLen characters + // over multiple lines. Default to 450 if not set. + SplitLen int +} + +// NewConfig creates a Config struct containing sensible defaults. +// It takes one required argument: the nick to use for the client. +// Subsequent string arguments set the client's ident and "real" +// name, but these are optional. +func NewConfig(nick string, args ...string) *Config { + cfg := &Config{ + Me: &state.Nick{Nick: nick}, + PingFreq: 3 * time.Minute, + NewNick: func(s string) string { return s + "_" }, + Recover: (*Conn).LogPanic, // in dispatch.go + SplitLen: defaultSplit, + Timeout: 60 * time.Second, + } + cfg.Me.Ident = "goirc" + if len(args) > 0 && args[0] != "" { + cfg.Me.Ident = args[0] + } + cfg.Me.Name = "Powered by GoIRC" + if len(args) > 1 && args[1] != "" { + cfg.Me.Name = args[1] + } + cfg.Version = "Powered by GoIRC" + cfg.QuitMessage = "GoBye!" + return cfg +} + +// SimpleClient creates a new Conn, passing its arguments to NewConfig. +// If you don't need to change any client options and just want to get +// started quickly, this is a convenient shortcut. +func SimpleClient(nick string, args ...string) *Conn { + conn := Client(NewConfig(nick, args...)) + return conn +} + +// Client takes a Config struct and returns a new Conn ready to have +// handlers added and connect to a server. +func Client(cfg *Config) *Conn { + if cfg == nil { + cfg = NewConfig("__idiot__") + } + if cfg.Me == nil || cfg.Me.Nick == "" || cfg.Me.Ident == "" { + cfg.Me = &state.Nick{Nick: "__idiot__"} + cfg.Me.Ident = "goirc" + cfg.Me.Name = "Powered by GoIRC" + } + + dialer := new(net.Dialer) + dialer.Timeout = cfg.Timeout + if cfg.LocalAddr != "" { + if !hasPort(cfg.LocalAddr) { + cfg.LocalAddr += ":0" + } + + local, err := net.ResolveTCPAddr("tcp", cfg.LocalAddr) + if err == nil { + dialer.LocalAddr = local + } else { + logging.Error("irc.Client(): Cannot resolve local address %s: %s", cfg.LocalAddr, err) + } + } + + conn := &Conn{ + cfg: cfg, + dialer: dialer, + intHandlers: handlerSet(), + fgHandlers: handlerSet(), + bgHandlers: handlerSet(), + stRemovers: make([]Remover, 0, len(stHandlers)), + lastsent: time.Now(), + } + conn.addIntHandlers() + return conn +} + +// Connected returns true if the client is successfully connected to +// an IRC server. It becomes true when the TCP connection is established, +// and false again when the connection is closed. +func (conn *Conn) Connected() bool { + conn.mu.RLock() + defer conn.mu.RUnlock() + return conn.connected +} + +// Config returns a pointer to the Config struct used by the client. +// Many of the elements of Config may be changed at any point to +// affect client behaviour. To disable flood protection temporarily, +// for example, a handler could do: +// +// conn.Config().Flood = true +// // Send many lines to the IRC server, risking "excess flood" +// conn.Config().Flood = false +// +func (conn *Conn) Config() *Config { + return conn.cfg +} + +// Me returns a state.Nick that reflects the client's IRC nick at the +// time it is called. If state tracking is enabled, this comes from +// the tracker, otherwise it is equivalent to conn.cfg.Me. +func (conn *Conn) Me() *state.Nick { + if conn.st != nil { + conn.cfg.Me = conn.st.Me() + } + return conn.cfg.Me +} + +// StateTracker returns the state tracker being used by the client, +// if tracking is enabled, and nil otherwise. +func (conn *Conn) StateTracker() state.Tracker { + return conn.st +} + +// EnableStateTracking causes the client to track information about +// all channels it is joined to, and all the nicks in those channels. +// This can be rather handy for a number of bot-writing tasks. See +// the state package for more details. +// +// NOTE: Calling this while connected to an IRC server may cause the +// state tracker to become very confused all over STDERR if logging +// is enabled. State tracking should enabled before connecting or +// at a pinch while the client is not joined to any channels. +func (conn *Conn) EnableStateTracking() { + conn.mu.Lock() + defer conn.mu.Unlock() + if conn.st == nil { + n := conn.cfg.Me + conn.st = state.NewTracker(n.Nick) + conn.st.NickInfo(n.Nick, n.Ident, n.Host, n.Name) + conn.cfg.Me = conn.st.Me() + conn.addSTHandlers() + } +} + +// DisableStateTracking causes the client to stop tracking information +// about the channels and nicks it knows of. It will also wipe current +// state from the state tracker. +func (conn *Conn) DisableStateTracking() { + conn.mu.Lock() + defer conn.mu.Unlock() + if conn.st != nil { + conn.cfg.Me = conn.st.Me() + conn.delSTHandlers() + conn.st.Wipe() + conn.st = nil + } +} + +// Per-connection state initialisation. +func (conn *Conn) initialise() { + conn.io = nil + conn.sock = nil + conn.in = make(chan *Line, 32) + conn.out = make(chan string, 32) + conn.die = make(chan struct{}) + if conn.st != nil { + conn.st.Wipe() + } +} + +// ConnectTo connects the IRC client to "host[:port]", which should be either +// a hostname or an IP address, with an optional port. It sets the client's +// Config.Server to host, Config.Pass to pass if one is provided, and then +// calls Connect. +func (conn *Conn) ConnectTo(host string, pass ...string) error { + conn.cfg.Server = host + if len(pass) > 0 { + conn.cfg.Pass = pass[0] + } + return conn.Connect() +} + +// Connect connects the IRC client to the server configured in Config.Server. +// To enable explicit SSL on the connection to the IRC server, set Config.SSL +// to true before calling Connect(). The port will default to 6697 if SSL is +// enabled, and 6667 otherwise. +// To enable connecting via a proxy server, set Config.Proxy to the proxy URL +// (example socks5://localhost:9000) before calling Connect(). +// +// Upon successful connection, Connected will return true and a REGISTER event +// will be fired. This is mostly for internal use; it is suggested that a +// handler for the CONNECTED event is used to perform any initial client work +// like joining channels and sending messages. +func (conn *Conn) Connect() error { + // We don't want to hold conn.mu while firing the REGISTER event, + // and it's much easier and less error prone to defer the unlock, + // so the connect mechanics have been delegated to internalConnect. + err := conn.internalConnect() + if err == nil { + conn.dispatch(&Line{Cmd: REGISTER, Time: time.Now()}) + } + return err +} + +// internalConnect handles the work of actually connecting to the server. +func (conn *Conn) internalConnect() error { + conn.mu.Lock() + defer conn.mu.Unlock() + conn.initialise() + + if conn.cfg.Server == "" { + return fmt.Errorf("irc.Connect(): cfg.Server must be non-empty") + } + if conn.connected { + return fmt.Errorf("irc.Connect(): Cannot connect to %s, already connected.", conn.cfg.Server) + } + + if !hasPort(conn.cfg.Server) { + if conn.cfg.SSL { + conn.cfg.Server = net.JoinHostPort(conn.cfg.Server, "6697") + } else { + conn.cfg.Server = net.JoinHostPort(conn.cfg.Server, "6667") + } + } + + if conn.cfg.Proxy != "" { + proxyURL, err := url.Parse(conn.cfg.Proxy) + if err != nil { + return err + } + conn.proxyDialer, err = proxy.FromURL(proxyURL, conn.dialer) + if err != nil { + return err + } + + logging.Info("irc.Connect(): Connecting to %s.", conn.cfg.Server) + if s, err := conn.proxyDialer.Dial("tcp", conn.cfg.Server); err == nil { + conn.sock = s + } else { + return err + } + } else { + logging.Info("irc.Connect(): Connecting to %s.", conn.cfg.Server) + if s, err := conn.dialer.Dial("tcp", conn.cfg.Server); err == nil { + conn.sock = s + } else { + return err + } + } + + if conn.cfg.SSL { + logging.Info("irc.Connect(): Performing SSL handshake.") + s := tls.Client(conn.sock, conn.cfg.SSLConfig) + if err := s.Handshake(); err != nil { + return err + } + conn.sock = s + } + + conn.postConnect(true) + conn.connected = true + return nil +} + +// postConnect performs post-connection setup, for ease of testing. +func (conn *Conn) postConnect(start bool) { + conn.io = bufio.NewReadWriter( + bufio.NewReader(conn.sock), + bufio.NewWriter(conn.sock)) + if start { + conn.wg.Add(3) + go conn.send() + go conn.recv() + go conn.runLoop() + if conn.cfg.PingFreq > 0 { + conn.wg.Add(1) + go conn.ping() + } + } +} + +// hasPort returns true if the string hostname has a :port suffix. +// It was copied from net/http for great justice. +func hasPort(s string) bool { + return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") +} + +// send is started as a goroutine after a connection is established. +// It shuttles data from the output channel to write(), and is killed +// when Conn.die is closed. +func (conn *Conn) send() { + for { + select { + case line := <-conn.out: + if err := conn.write(line); err != nil { + logging.Error("irc.send(): %s", err.Error()) + // We can't defer this, because Close() waits for it. + conn.wg.Done() + conn.Close() + return + } + case <-conn.die: + // control channel closed, bail out + conn.wg.Done() + return + } + } +} + +// recv is started as a goroutine after a connection is established. +// It receives "\r\n" terminated lines from the server, parses them into +// Lines, and sends them to the input channel. +func (conn *Conn) recv() { + for { + s, err := conn.io.ReadString('\n') + if err != nil { + if err != io.EOF { + logging.Error("irc.recv(): %s", err.Error()) + } + // We can't defer this, because Close() waits for it. + conn.wg.Done() + conn.Close() + return + } + s = strings.Trim(s, "\r\n") + logging.Debug("<- %s", s) + + if line := ParseLine(s); line != nil { + line.Time = time.Now() + conn.in <- line + } else { + logging.Warn("irc.recv(): problems parsing line:\n %s", s) + } + } +} + +// ping is started as a goroutine after a connection is established, as +// long as Config.PingFreq >0. It pings the server every PingFreq seconds. +func (conn *Conn) ping() { + defer conn.wg.Done() + tick := time.NewTicker(conn.cfg.PingFreq) + for { + select { + case <-tick.C: + conn.Ping(fmt.Sprintf("%d", time.Now().UnixNano())) + case <-conn.die: + // control channel closed, bail out + tick.Stop() + return + } + } +} + +// runLoop is started as a goroutine after a connection is established. +// It pulls Lines from the input channel and dispatches them to any +// handlers that have been registered for that IRC verb. +func (conn *Conn) runLoop() { + defer conn.wg.Done() + for { + select { + case line := <-conn.in: + conn.dispatch(line) + case <-conn.die: + // control channel closed, bail out + return + } + } +} + +// write writes a \r\n terminated line of output to the connected server, +// using Hybrid's algorithm to rate limit if conn.cfg.Flood is false. +func (conn *Conn) write(line string) error { + if !conn.cfg.Flood { + if t := conn.rateLimit(len(line)); t != 0 { + // sleep for the current line's time value before sending it + logging.Info("irc.rateLimit(): Flood! Sleeping for %.2f secs.", + t.Seconds()) + <-time.After(t) + } + } + + if _, err := conn.io.WriteString(line + "\r\n"); err != nil { + return err + } + if err := conn.io.Flush(); err != nil { + return err + } + if strings.HasPrefix(line, "PASS") { + line = "PASS **************" + } + logging.Debug("-> %s", line) + return nil +} + +// rateLimit implements Hybrid's flood control algorithm for outgoing lines. +func (conn *Conn) rateLimit(chars int) time.Duration { + // Hybrid's algorithm allows for 2 seconds per line and an additional + // 1/120 of a second per character on that line. + linetime := 2*time.Second + time.Duration(chars)*time.Second/120 + elapsed := time.Now().Sub(conn.lastsent) + if conn.badness += linetime - elapsed; conn.badness < 0 { + // negative badness times are badness... + conn.badness = 0 + } + conn.lastsent = time.Now() + // If we've sent more than 10 second's worth of lines according to the + // calculation above, then we're at risk of "Excess Flood". + if conn.badness > 10*time.Second { + return linetime + } + return 0 +} + +// Close tears down all connection-related state. It is called when either +// the sending or receiving goroutines encounter an error. +// It may also be used to forcibly shut down the connection to the server. +func (conn *Conn) Close() error { + // Guard against double-call of Close() if we get an error in send() + // as calling sock.Close() will cause recv() to receive EOF in readstring() + conn.mu.Lock() + if !conn.connected { + conn.mu.Unlock() + return nil + } + logging.Info("irc.Close(): Disconnected from server.") + conn.connected = false + err := conn.sock.Close() + close(conn.die) + // Drain both in and out channels to avoid a deadlock if the buffers + // have filled. See TestSendDeadlockOnFullBuffer in connection_test.go. + conn.drainIn() + conn.drainOut() + conn.wg.Wait() + conn.mu.Unlock() + // Dispatch after closing connection but before reinit + // so event handlers can still access state information. + conn.dispatch(&Line{Cmd: DISCONNECTED, Time: time.Now()}) + return err +} + +// drainIn sends all data buffered in conn.in to /dev/null. +func (conn *Conn) drainIn() { + for { + select { + case <-conn.in: + default: + return + } + } +} + +// drainOut does the same for conn.out. Generics! +func (conn *Conn) drainOut() { + for { + select { + case <-conn.out: + default: + return + } + } +} + +// Dumps a load of information about the current state of the connection to a +// string for debugging state tracking and other such things. +func (conn *Conn) String() string { + str := "GoIRC Connection\n" + str += "----------------\n\n" + if conn.Connected() { + str += "Connected to " + conn.cfg.Server + "\n\n" + } else { + str += "Not currently connected!\n\n" + } + str += conn.Me().String() + "\n" + if conn.st != nil { + str += conn.st.String() + "\n" + } + return str +} diff --git a/vendor/github.com/fluffle/goirc/client/connection_test.go b/vendor/github.com/fluffle/goirc/client/connection_test.go new file mode 100644 index 0000000..acf4713 --- /dev/null +++ b/vendor/github.com/fluffle/goirc/client/connection_test.go @@ -0,0 +1,585 @@ +package client + +import ( + "runtime" + "strings" + "testing" + "time" + + "github.com/fluffle/goirc/state" + "github.com/golang/mock/gomock" +) + +type checker struct { + t *testing.T + c chan struct{} +} + +func callCheck(t *testing.T) checker { + return checker{t: t, c: make(chan struct{})} +} + +func (c checker) call() { + c.c <- struct{}{} +} + +func (c checker) assertNotCalled(fmt string, args ...interface{}) { + select { + case <-c.c: + c.t.Errorf(fmt, args...) + default: + } +} + +func (c checker) assertWasCalled(fmt string, args ...interface{}) { + select { + case <-c.c: + case <-time.After(time.Millisecond): + // Usually need to wait for goroutines to settle :-/ + c.t.Errorf(fmt, args...) + } +} + +type testState struct { + ctrl *gomock.Controller + st *state.MockTracker + nc *mockNetConn + c *Conn +} + +// NOTE: including a second argument at all prevents calling c.postConnect() +func setUp(t *testing.T, start ...bool) (*Conn, *testState) { + ctrl := gomock.NewController(t) + st := state.NewMockTracker(ctrl) + nc := MockNetConn(t) + c := SimpleClient("test", "test", "Testing IRC") + c.initialise() + + c.st = st + c.sock = nc + c.cfg.Flood = true // Tests can take a while otherwise + c.connected = true + // If a second argument is passed to setUp, we tell postConnect not to + // start the various goroutines that shuttle data around. + c.postConnect(len(start) == 0) + // Sleep 1ms to allow background routines to start. + <-time.After(time.Millisecond) + + return c, &testState{ctrl, st, nc, c} +} + +func (s *testState) tearDown() { + s.nc.ExpectNothing() + s.c.Close() + s.ctrl.Finish() +} + +// Practically the same as the above test, but Close is called implicitly +// by recv() getting an EOF from the mock connection. +func TestEOF(t *testing.T) { + c, s := setUp(t) + // Since we're not using tearDown() here, manually call Finish() + defer s.ctrl.Finish() + + // Set up a handler to detect whether disconnected handlers are called + dcon := callCheck(t) + c.HandleFunc(DISCONNECTED, func(conn *Conn, line *Line) { + dcon.call() + }) + + // Simulate EOF from server + s.nc.Close() + + // Verify that disconnected handler was called + dcon.assertWasCalled("Conn did not call disconnected handlers.") + + // Verify that the connection no longer thinks it's connected + if c.Connected() { + t.Errorf("Conn still thinks it's connected to the server.") + } +} + +func TestClientAndStateTracking(t *testing.T) { + ctrl := gomock.NewController(t) + st := state.NewMockTracker(ctrl) + c := SimpleClient("test", "test", "Testing IRC") + + // Assert some basic things about the initial state of the Conn struct + me := c.cfg.Me + if me.Nick != "test" || me.Ident != "test" || + me.Name != "Testing IRC" || me.Host != "" { + t.Errorf("Conn.cfg.Me not correctly initialised.") + } + // Check that the internal handlers are correctly set up + for k, _ := range intHandlers { + if _, ok := c.intHandlers.set[strings.ToLower(k)]; !ok { + t.Errorf("Missing internal handler for '%s'.", k) + } + } + + // Now enable the state tracking code and check its handlers + c.EnableStateTracking() + for k, _ := range stHandlers { + if _, ok := c.intHandlers.set[strings.ToLower(k)]; !ok { + t.Errorf("Missing state handler for '%s'.", k) + } + } + if len(c.stRemovers) != len(stHandlers) { + t.Errorf("Incorrect number of Removers (%d != %d) when adding state handlers.", + len(c.stRemovers), len(stHandlers)) + } + if neu := c.Me(); neu.Nick != me.Nick || neu.Ident != me.Ident || + neu.Name != me.Name || neu.Host != me.Host { + t.Errorf("Enabling state tracking erased information about me!") + } + + // We're expecting the untracked me to be replaced by a tracked one + if c.st == nil { + t.Errorf("State tracker not enabled correctly.") + } + if me = c.cfg.Me; me.Nick != "test" || me.Ident != "test" || + me.Name != "Testing IRC" || me.Host != "" { + t.Errorf("Enabling state tracking did not replace Me correctly.") + } + + // Now, shim in the mock state tracker and test disabling state tracking + c.st = st + gomock.InOrder( + st.EXPECT().Me().Return(me), + st.EXPECT().Wipe(), + ) + c.DisableStateTracking() + if c.st != nil || !c.cfg.Me.Equals(me) { + t.Errorf("State tracker not disabled correctly.") + } + + // Finally, check state tracking handlers were all removed correctly + for k, _ := range stHandlers { + if _, ok := c.intHandlers.set[strings.ToLower(k)]; ok && k != "NICK" { + // A bit leaky, because intHandlers adds a NICK handler. + t.Errorf("State handler for '%s' not removed correctly.", k) + } + } + if len(c.stRemovers) != 0 { + t.Errorf("stRemovers not zeroed correctly when removing state handlers.") + } + ctrl.Finish() +} + +func TestSendExitsOnDie(t *testing.T) { + // Passing a second value to setUp stops goroutines from starting + c, s := setUp(t, false) + defer s.tearDown() + + // Assert that before send is running, nothing should be sent to the socket + // but writes to the buffered channel "out" should not block. + c.out <- "SENT BEFORE START" + s.nc.ExpectNothing() + + // We want to test that the a goroutine calling send will exit correctly. + exited := callCheck(t) + // send() will decrement the WaitGroup, so we must increment it. + c.wg.Add(1) + go func() { + c.send() + exited.call() + }() + + // send is now running in the background as if started by postConnect. + // This should read the line previously buffered in c.out, and write it + // to the socket connection. + s.nc.Expect("SENT BEFORE START") + + // Send another line, just to be sure :-) + c.out <- "SENT AFTER START" + s.nc.Expect("SENT AFTER START") + + // Now, use the control channel to exit send and kill the goroutine. + // This sneakily uses the fact that the other two goroutines that would + // normally be waiting for die to close are not running, so we only send + // to the goroutine started above. Normally Close() closes c.die and + // signals to all three goroutines (send, ping, runLoop) to exit. + exited.assertNotCalled("Exited before signal sent.") + c.die <- struct{}{} + exited.assertWasCalled("Didn't exit after signal.") + s.nc.ExpectNothing() + + // Sending more on c.out shouldn't reach the network. + c.out <- "SENT AFTER END" + s.nc.ExpectNothing() +} + +func TestSendExitsOnWriteError(t *testing.T) { + // Passing a second value to setUp stops goroutines from starting + c, s := setUp(t, false) + // We can't use tearDown here because we're testing shutdown conditions + // (and so need to EXPECT() a call to st.Wipe() in the right place) + defer s.ctrl.Finish() + + // We want to test that the a goroutine calling send will exit correctly. + exited := callCheck(t) + // send() will decrement the WaitGroup, so we must increment it. + c.wg.Add(1) + go func() { + c.send() + exited.call() + }() + + // Send a line to be sure things are good. + c.out <- "SENT AFTER START" + s.nc.Expect("SENT AFTER START") + + // Now, close the underlying socket to cause write() to return an error. + // This will call Close() => a call to st.Wipe() will happen. + exited.assertNotCalled("Exited before signal sent.") + s.nc.Close() + // Sending more on c.out shouldn't reach the network, but we need to send + // *something* to trigger a call to write() that will fail. + c.out <- "SENT AFTER END" + exited.assertWasCalled("Didn't exit after signal.") + s.nc.ExpectNothing() +} + +func TestSendDeadlockOnFullBuffer(t *testing.T) { + // Passing a second value to setUp stops goroutines from starting + c, s := setUp(t, false) + // We can't use tearDown here because we're testing a deadlock condition + // and if tearDown tries to call Close() it will deadlock some more + // because send() is holding the conn mutex via Close() already. + defer s.ctrl.Finish() + + // We want to test that the a goroutine calling send will exit correctly. + loopExit := callCheck(t) + sendExit := callCheck(t) + // send() and runLoop() will decrement the WaitGroup, so we must increment it. + c.wg.Add(2) + + // The deadlock arises when a handler being called from conn.dispatch() in + // runLoop() tries to write to conn.out to send a message back to the IRC + // server, but the buffer is full. If at the same time send() is + // calling conn.Close() and waiting in there for runLoop() to call + // conn.wg.Done(), it will not empty the buffer of conn.out => deadlock. + // + // We simulate this by artifically filling conn.out. We must use a + // goroutine to put in one more line than the buffer can hold, because + // send() will read a line from conn.out on its first loop iteration: + go func() { + for i := 0; i < 33; i++ { + c.out <- "FILL BUFFER WITH CRAP" + } + }() + // Then we add a handler that tries to write a line to conn.out: + c.HandleFunc(PRIVMSG, func(conn *Conn, line *Line) { + conn.Raw(line.Raw) + }) + // And trigger it by starting runLoop and inserting a line into conn.in: + go func() { + c.runLoop() + loopExit.call() + }() + c.in <- &Line{Cmd: PRIVMSG, Raw: "WRITE THAT CAUSES DEADLOCK"} + + // At this point the handler should be blocked on a write to conn.out, + // preventing runLoop from looping and thus noticing conn.die is closed. + // + // The next part is to force send() to call conn.Close(), which can + // be done by closing the fake net.Conn so that it returns an error on + // calls to Write(): + s.nc.ExpectNothing() + s.nc.Close() + + // Now when send is started it will read one line from conn.out and try + // to write it to the socket. It should immediately receive an error and + // call conn.Close(), triggering the deadlock as it waits forever for + // runLoop to call conn.wg.Done. + go func() { + c.send() + sendExit.call() + }() + + // Make sure that things are definitely deadlocked. + <-time.After(time.Millisecond) + + // Verify that the connection no longer thinks it's connected, i.e. + // conn.Close() has definitely been called. We can't call + // conn.Connected() here because conn.Close() holds the mutex. + if c.connected { + t.Errorf("Conn still thinks it's connected to the server.") + } + + // We expect both loops to terminate cleanly. If either of them don't + // then we have successfully deadlocked :-( + loopExit.assertWasCalled("runLoop did not exit cleanly.") + sendExit.assertWasCalled("send did not exit cleanly.") +} + +func TestRecv(t *testing.T) { + // Passing a second value to setUp stops goroutines from starting + c, s := setUp(t, false) + // We can't use tearDown here because we're testing shutdown conditions + // (and so need to EXPECT() a call to st.Wipe() in the right place) + defer s.ctrl.Finish() + + // Send a line before recv is started up, to verify nothing appears on c.in + s.nc.Send(":irc.server.org 001 test :First test line.") + + // reader is a helper to do a "non-blocking" read of c.in + reader := func() *Line { + select { + case <-time.After(time.Millisecond): + case l := <-c.in: + return l + } + return nil + } + if l := reader(); l != nil { + t.Errorf("Line parsed before recv started.") + } + + // We want to test that the a goroutine calling recv will exit correctly. + exited := callCheck(t) + // recv() will decrement the WaitGroup, so we must increment it. + c.wg.Add(1) + go func() { + c.recv() + exited.call() + }() + + // Now, this should mean that we'll receive our parsed line on c.in + if l := reader(); l == nil || l.Cmd != "001" { + t.Errorf("Bad first line received on input channel") + } + + // Send a second line, just to be sure. + s.nc.Send(":irc.server.org 002 test :Second test line.") + if l := reader(); l == nil || l.Cmd != "002" { + t.Errorf("Bad second line received on input channel.") + } + + // Test that recv does something useful with a line it can't parse + // (not that there are many, ParseLine is forgiving). + s.nc.Send(":textwithnospaces") + if l := reader(); l != nil { + t.Errorf("Bad line still caused receive on input channel.") + } + + // The only way recv() exits is when the socket closes. + exited.assertNotCalled("Exited before socket close.") + s.nc.Close() + exited.assertWasCalled("Didn't exit on socket close.") + + // Since s.nc is closed we can't attempt another send on it... + if l := reader(); l != nil { + t.Errorf("Line received on input channel after socket close.") + } +} + +func TestPing(t *testing.T) { + // Passing a second value to setUp stops goroutines from starting + c, s := setUp(t, false) + defer s.tearDown() + + res := time.Millisecond + + // Windows has a timer resolution of 15.625ms by default. + // This means the test will be slower on windows, but + // should at least stop most of the flakiness... + // https://github.com/fluffle/goirc/issues/88 + if runtime.GOOS == "windows" { + res = 15625 * time.Microsecond + } + + // Set a low ping frequency for testing. + c.cfg.PingFreq = 10 * res + + // reader is a helper to do a "non-blocking" read of c.out + reader := func() string { + select { + case <-time.After(res): + case s := <-c.out: + return s + } + return "" + } + if s := reader(); s != "" { + t.Errorf("Line output before ping started.") + } + + // Start ping loop. + exited := callCheck(t) + // ping() will decrement the WaitGroup, so we must increment it. + c.wg.Add(1) + go func() { + c.ping() + exited.call() + }() + + // The first ping should be after 10*res ms, + // so we don't expect anything now on c.in + if s := reader(); s != "" { + t.Errorf("Line output directly after ping started.") + } + + <-time.After(c.cfg.PingFreq) + if s := reader(); s == "" || !strings.HasPrefix(s, "PING :") { + t.Errorf("Line not output after %s.", c.cfg.PingFreq) + } + + // Reader waits for res ms and we call it a few times above. + <-time.After(7 * res) + if s := reader(); s != "" { + t.Errorf("Line output <%s after last ping.", 7*res) + } + + // This is a short window in which the ping should happen + // This may result in flaky tests; sorry (and file a bug) if so. + <-time.After(2 * res) + if s := reader(); s == "" || !strings.HasPrefix(s, "PING :") { + t.Errorf("Line not output after another %s.", 2*res) + } + + // Now kill the ping loop. + // This sneakily uses the fact that the other two goroutines that would + // normally be waiting for die to close are not running, so we only send + // to the goroutine started above. Normally Close() closes c.die and + // signals to all three goroutines (send, ping, runLoop) to exit. + exited.assertNotCalled("Exited before signal sent.") + c.die <- struct{}{} + exited.assertWasCalled("Didn't exit after signal.") + // Make sure we're no longer pinging by waiting >2x PingFreq + <-time.After(2*c.cfg.PingFreq + res) + if s := reader(); s != "" { + t.Errorf("Line output after ping stopped.") + } +} + +func TestRunLoop(t *testing.T) { + // Passing a second value to setUp stops goroutines from starting + c, s := setUp(t, false) + defer s.tearDown() + + // Set up a handler to detect whether 001 handler is called + h001 := callCheck(t) + c.HandleFunc("001", func(conn *Conn, line *Line) { + h001.call() + }) + h002 := callCheck(t) + // Set up a handler to detect whether 002 handler is called + c.HandleFunc("002", func(conn *Conn, line *Line) { + h002.call() + }) + + l1 := ParseLine(":irc.server.org 001 test :First test line.") + c.in <- l1 + h001.assertNotCalled("001 handler called before runLoop started.") + + // We want to test that the a goroutine calling runLoop will exit correctly. + // Now, we can expect the call to Dispatch to take place as runLoop starts. + exited := callCheck(t) + // runLoop() will decrement the WaitGroup, so we must increment it. + c.wg.Add(1) + go func() { + c.runLoop() + exited.call() + }() + h001.assertWasCalled("001 handler not called after runLoop started.") + + // Send another line, just to be sure :-) + h002.assertNotCalled("002 handler called before expected.") + l2 := ParseLine(":irc.server.org 002 test :Second test line.") + c.in <- l2 + h002.assertWasCalled("002 handler not called while runLoop started.") + + // Now, use the control channel to exit send and kill the goroutine. + // This sneakily uses the fact that the other two goroutines that would + // normally be waiting for die to close are not running, so we only send + // to the goroutine started above. Normally Close() closes c.die and + // signals to all three goroutines (send, ping, runLoop) to exit. + exited.assertNotCalled("Exited before signal sent.") + c.die <- struct{}{} + exited.assertWasCalled("Didn't exit after signal.") + + // Sending more on c.in shouldn't dispatch any further events + c.in <- l1 + h001.assertNotCalled("001 handler called after runLoop ended.") +} + +func TestWrite(t *testing.T) { + // Passing a second value to setUp stops goroutines from starting + c, s := setUp(t, false) + // We can't use tearDown here because we're testing shutdown conditions + // (and so need to EXPECT() a call to st.Wipe() in the right place) + defer s.ctrl.Finish() + + // Write should just write a line to the socket. + if err := c.write("yo momma"); err != nil { + t.Errorf("Write returned unexpected error %v", err) + } + s.nc.Expect("yo momma") + + // Flood control is disabled -- setUp sets c.cfg.Flood = true -- so we should + // not have set c.badness at this point. + if c.badness != 0 { + t.Errorf("Flood control used when Flood = true.") + } + + c.cfg.Flood = false + if err := c.write("she so useless"); err != nil { + t.Errorf("Write returned unexpected error %v", err) + } + s.nc.Expect("she so useless") + + // The lastsent time should have been updated very recently... + if time.Now().Sub(c.lastsent) > time.Millisecond { + t.Errorf("Flood control not used when Flood = false.") + } + + // Finally, test the error state by closing the socket then writing. + s.nc.Close() + if err := c.write("she can't pass unit tests"); err == nil { + t.Errorf("Expected write to return error after socket close.") + } +} + +func TestRateLimit(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + if c.badness != 0 { + t.Errorf("Bad initial values for rate limit variables.") + } + + // We'll be needing this later... + abs := func(i time.Duration) time.Duration { + if i < 0 { + return -i + } + return i + } + + // Since the changes to the time module, c.lastsent is now a time.Time. + // It's initialised on client creation to time.Now() which for the purposes + // of this test was probably around 1.2 ms ago. This is inconvenient. + // Making it >10s ago effectively clears out the inconsistency, as this + // makes elapsed > linetime and thus zeros c.badness and resets c.lastsent. + c.lastsent = time.Now().Add(-10 * time.Second) + if l := c.rateLimit(60); l != 0 || c.badness != 0 { + t.Errorf("Rate limit got non-zero badness from long-ago lastsent.") + } + + // So, time at the nanosecond resolution is a bit of a bitch. Choosing 60 + // characters as the line length means we should be increasing badness by + // 2.5 seconds minus the delta between the two ratelimit calls. This should + // be minimal but it's guaranteed that it won't be zero. Use 20us as a fuzz. + if l := c.rateLimit(60); l != 0 || + abs(c.badness-2500*time.Millisecond) > 20*time.Microsecond { + t.Errorf("Rate limit calculating badness incorrectly.") + } + // At this point, we can tip over the badness scale, with a bit of help. + // 720 chars => +8 seconds of badness => 10.5 seconds => ratelimit + if l := c.rateLimit(720); l != 8*time.Second || + abs(c.badness-10500*time.Millisecond) > 20*time.Microsecond { + t.Errorf("Rate limit failed to return correct limiting values.") + t.Errorf("l=%d, badness=%d", l, c.badness) + } +} diff --git a/vendor/github.com/fluffle/goirc/client/dispatch.go b/vendor/github.com/fluffle/goirc/client/dispatch.go new file mode 100644 index 0000000..4a5deb9 --- /dev/null +++ b/vendor/github.com/fluffle/goirc/client/dispatch.go @@ -0,0 +1,202 @@ +package client + +import ( + "runtime" + "strings" + "sync" + + "github.com/fluffle/goirc/logging" +) + +// Handlers are triggered on incoming Lines from the server, with the handler +// "name" being equivalent to Line.Cmd. Read the RFCs for details on what +// replies could come from the server. They'll generally be things like +// "PRIVMSG", "JOIN", etc. but all the numeric replies are left as ascii +// strings of digits like "332" (mainly because I really didn't feel like +// putting massive constant tables in). +// +// Foreground handlers have a guarantee of protocol consistency: all the +// handlers for one event will have finished before the handlers for the +// next start processing. They are run in parallel but block the event +// loop, so care should be taken to ensure these handlers are quick :-) +// +// Background handlers are run in parallel and do not block the event loop. +// This is useful for things that may need to do significant work. +type Handler interface { + Handle(*Conn, *Line) +} + +// Removers allow for a handler that has been previously added to the client +// to be removed. +type Remover interface { + Remove() +} + +// HandlerFunc allows a bare function with this signature to implement the +// Handler interface. It is used by Conn.HandleFunc. +type HandlerFunc func(*Conn, *Line) + +func (hf HandlerFunc) Handle(conn *Conn, line *Line) { + hf(conn, line) +} + +// Handlers are organised using a map of linked-lists, with each map +// key representing an IRC verb or numeric, and the linked list values +// being handlers that are executed in parallel when a Line from the +// server with that verb or numeric arrives. +type hSet struct { + set map[string]*hList + sync.RWMutex +} + +type hList struct { + start, end *hNode +} + +// Storing the forward and backward links in the node allows O(1) removal. +// This probably isn't strictly necessary but I think it's kinda nice. +type hNode struct { + next, prev *hNode + set *hSet + event string + handler Handler +} + +// A hNode implements both Handler (with configurable panic recovery)... +func (hn *hNode) Handle(conn *Conn, line *Line) { + defer conn.cfg.Recover(conn, line) + hn.handler.Handle(conn, line) +} + +// ... and Remover. +func (hn *hNode) Remove() { + hn.set.remove(hn) +} + +func handlerSet() *hSet { + return &hSet{set: make(map[string]*hList)} +} + +// When a new Handler is added for an event, it is wrapped in a hNode and +// returned as a Remover so the caller can remove it at a later time. +func (hs *hSet) add(ev string, h Handler) Remover { + hs.Lock() + defer hs.Unlock() + ev = strings.ToLower(ev) + l, ok := hs.set[ev] + if !ok { + l = &hList{} + } + hn := &hNode{ + set: hs, + event: ev, + handler: h, + } + if !ok { + l.start = hn + } else { + hn.prev = l.end + l.end.next = hn + } + l.end = hn + hs.set[ev] = l + return hn +} + +func (hs *hSet) remove(hn *hNode) { + hs.Lock() + defer hs.Unlock() + l, ok := hs.set[hn.event] + if !ok { + logging.Error("Removing node for unknown event '%s'", hn.event) + return + } + if hn.next == nil { + l.end = hn.prev + } else { + hn.next.prev = hn.prev + } + if hn.prev == nil { + l.start = hn.next + } else { + hn.prev.next = hn.next + } + hn.next = nil + hn.prev = nil + hn.set = nil + if l.start == nil || l.end == nil { + delete(hs.set, hn.event) + } +} + +func (hs *hSet) getHandlers(ev string) []*hNode { + hs.RLock() + defer hs.RUnlock() + list, ok := hs.set[ev] + if !ok { + return nil + } + // Copy current list of handlers to a temporary slice under the lock. + handlers := make([]*hNode, 0) + for hn := list.start; hn != nil; hn = hn.next { + handlers = append(handlers, hn) + } + return handlers +} + +func (hs *hSet) dispatch(conn *Conn, line *Line) { + ev := strings.ToLower(line.Cmd) + wg := &sync.WaitGroup{} + for _, hn := range hs.getHandlers(ev) { + wg.Add(1) + go func(hn *hNode) { + hn.Handle(conn, line.Copy()) + wg.Done() + }(hn) + } + wg.Wait() +} + +// Handle adds the provided handler to the foreground set for the named event. +// It will return a Remover that allows that handler to be removed again. +func (conn *Conn) Handle(name string, h Handler) Remover { + return conn.fgHandlers.add(name, h) +} + +// HandleBG adds the provided handler to the background set for the named +// event. It may go away in the future. +// It will return a Remover that allows that handler to be removed again. +func (conn *Conn) HandleBG(name string, h Handler) Remover { + return conn.bgHandlers.add(name, h) +} + +func (conn *Conn) handle(name string, h Handler) Remover { + return conn.intHandlers.add(name, h) +} + +// HandleFunc adds the provided function as a handler in the foreground set +// for the named event. +// It will return a Remover that allows that handler to be removed again. +func (conn *Conn) HandleFunc(name string, hf HandlerFunc) Remover { + return conn.Handle(name, hf) +} + +func (conn *Conn) dispatch(line *Line) { + // We run the internal handlers first, including all state tracking ones. + // This ensures that user-supplied handlers that use the tracker have a + // consistent view of the connection state in handlers that mutate it. + conn.intHandlers.dispatch(conn, line) + go conn.bgHandlers.dispatch(conn, line) + conn.fgHandlers.dispatch(conn, line) +} + +// LogPanic is used as the default panic catcher for the client. If, like me, +// you are not good with computer, and you'd prefer your bot not to vanish into +// the ether whenever you make unfortunate programming mistakes, you may find +// this useful: it will recover panics from handler code and log the errors. +func (conn *Conn) LogPanic(line *Line) { + if err := recover(); err != nil { + _, f, l, _ := runtime.Caller(2) + logging.Error("%s:%d: panic: %v", f, l, err) + } +} diff --git a/vendor/github.com/fluffle/goirc/client/dispatch_test.go b/vendor/github.com/fluffle/goirc/client/dispatch_test.go new file mode 100644 index 0000000..b79df64 --- /dev/null +++ b/vendor/github.com/fluffle/goirc/client/dispatch_test.go @@ -0,0 +1,201 @@ +package client + +import ( + "sync/atomic" + "testing" + "time" +) + +func TestHandlerSet(t *testing.T) { + // A Conn is needed here because the previous behaviour of passing nil to + // hset.dispatch causes a nil pointer dereference with panic recovery. + c, s := setUp(t) + defer s.tearDown() + + hs := handlerSet() + if len(hs.set) != 0 { + t.Errorf("New set contains things!") + } + + callcount := new(int32) + f := func(_ *Conn, _ *Line) { + atomic.AddInt32(callcount, 1) + } + + // Add one + hn1 := hs.add("ONE", HandlerFunc(f)).(*hNode) + hl, ok := hs.set["one"] + if len(hs.set) != 1 || !ok { + t.Errorf("Set doesn't contain 'one' list after add().") + } + if hn1.set != hs || hn1.event != "one" || hn1.prev != nil || hn1.next != nil { + t.Errorf("First node for 'one' not created correctly") + } + if hl.start != hn1 || hl.end != hn1 { + t.Errorf("Node not added to empty 'one' list correctly.") + } + + // Add another one... + hn2 := hs.add("one", HandlerFunc(f)).(*hNode) + if len(hs.set) != 1 { + t.Errorf("Set contains more than 'one' list after add().") + } + if hn2.set != hs || hn2.event != "one" { + t.Errorf("Second node for 'one' not created correctly") + } + if hn1.prev != nil || hn1.next != hn2 || hn2.prev != hn1 || hn2.next != nil { + t.Errorf("Nodes for 'one' not linked correctly.") + } + if hl.start != hn1 || hl.end != hn2 { + t.Errorf("Node not appended to 'one' list correctly.") + } + + // Add a third one! + hn3 := hs.add("one", HandlerFunc(f)).(*hNode) + if len(hs.set) != 1 { + t.Errorf("Set contains more than 'one' list after add().") + } + if hn3.set != hs || hn3.event != "one" { + t.Errorf("Third node for 'one' not created correctly") + } + if hn1.prev != nil || hn1.next != hn2 || + hn2.prev != hn1 || hn2.next != hn3 || + hn3.prev != hn2 || hn3.next != nil { + t.Errorf("Nodes for 'one' not linked correctly.") + } + if hl.start != hn1 || hl.end != hn3 { + t.Errorf("Node not appended to 'one' list correctly.") + } + + // And finally a fourth one! + hn4 := hs.add("one", HandlerFunc(f)).(*hNode) + if len(hs.set) != 1 { + t.Errorf("Set contains more than 'one' list after add().") + } + if hn4.set != hs || hn4.event != "one" { + t.Errorf("Fourth node for 'one' not created correctly.") + } + if hn1.prev != nil || hn1.next != hn2 || + hn2.prev != hn1 || hn2.next != hn3 || + hn3.prev != hn2 || hn3.next != hn4 || + hn4.prev != hn3 || hn4.next != nil { + t.Errorf("Nodes for 'one' not linked correctly.") + } + if hl.start != hn1 || hl.end != hn4 { + t.Errorf("Node not appended to 'one' list correctly.") + } + + // Dispatch should result in 4 additions. + if atomic.LoadInt32(callcount) != 0 { + t.Errorf("Something incremented call count before we were expecting it.") + } + hs.dispatch(c, &Line{Cmd: "One"}) + <-time.After(time.Millisecond) + if atomic.LoadInt32(callcount) != 4 { + t.Errorf("Our handler wasn't called four times :-(") + } + + // Remove node 3. + hn3.Remove() + if len(hs.set) != 1 { + t.Errorf("Set list count changed after remove().") + } + if hn3.set != nil || hn3.prev != nil || hn3.next != nil { + t.Errorf("Third node for 'one' not removed correctly.") + } + if hn1.prev != nil || hn1.next != hn2 || + hn2.prev != hn1 || hn2.next != hn4 || + hn4.prev != hn2 || hn4.next != nil { + t.Errorf("Third node for 'one' not unlinked correctly.") + } + if hl.start != hn1 || hl.end != hn4 { + t.Errorf("Third node for 'one' changed list pointers.") + } + + // Dispatch should result in 3 additions. + hs.dispatch(c, &Line{Cmd: "One"}) + <-time.After(time.Millisecond) + if atomic.LoadInt32(callcount) != 7 { + t.Errorf("Our handler wasn't called three times :-(") + } + + // Remove node 1. + hs.remove(hn1) + if len(hs.set) != 1 { + t.Errorf("Set list count changed after remove().") + } + if hn1.set != nil || hn1.prev != nil || hn1.next != nil { + t.Errorf("First node for 'one' not removed correctly.") + } + if hn2.prev != nil || hn2.next != hn4 || hn4.prev != hn2 || hn4.next != nil { + t.Errorf("First node for 'one' not unlinked correctly.") + } + if hl.start != hn2 || hl.end != hn4 { + t.Errorf("First node for 'one' didn't change list pointers.") + } + + // Dispatch should result in 2 additions. + hs.dispatch(c, &Line{Cmd: "One"}) + <-time.After(time.Millisecond) + if atomic.LoadInt32(callcount) != 9 { + t.Errorf("Our handler wasn't called two times :-(") + } + + // Remove node 4. + hn4.Remove() + if len(hs.set) != 1 { + t.Errorf("Set list count changed after remove().") + } + if hn4.set != nil || hn4.prev != nil || hn4.next != nil { + t.Errorf("Fourth node for 'one' not removed correctly.") + } + if hn2.prev != nil || hn2.next != nil { + t.Errorf("Fourth node for 'one' not unlinked correctly.") + } + if hl.start != hn2 || hl.end != hn2 { + t.Errorf("Fourth node for 'one' didn't change list pointers.") + } + + // Dispatch should result in 1 addition. + hs.dispatch(c, &Line{Cmd: "One"}) + <-time.After(time.Millisecond) + if atomic.LoadInt32(callcount) != 10 { + t.Errorf("Our handler wasn't called once :-(") + } + + // Remove node 2. + hs.remove(hn2) + if len(hs.set) != 0 { + t.Errorf("Removing last node in 'one' didn't remove list.") + } + if hn2.set != nil || hn2.prev != nil || hn2.next != nil { + t.Errorf("Second node for 'one' not removed correctly.") + } + if hl.start != nil || hl.end != nil { + t.Errorf("Second node for 'one' didn't change list pointers.") + } + + // Dispatch should result in NO additions. + hs.dispatch(c, &Line{Cmd: "One"}) + <-time.After(time.Millisecond) + if atomic.LoadInt32(callcount) != 10 { + t.Errorf("Our handler was called?") + } +} + +func TestPanicRecovery(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + recovered := callCheck(t) + c.cfg.Recover = func(conn *Conn, line *Line) { + if err, ok := recover().(string); ok && err == "panic!" { + recovered.call() + } + } + c.HandleFunc(PRIVMSG, func(conn *Conn, line *Line) { + panic("panic!") + }) + c.in <- ParseLine(":nick!user@host.com PRIVMSG #channel :OH NO PIGEONS") + recovered.assertWasCalled("Failed to recover panic!") +} diff --git a/vendor/github.com/fluffle/goirc/client/doc.go b/vendor/github.com/fluffle/goirc/client/doc.go new file mode 100644 index 0000000..08a79dd --- /dev/null +++ b/vendor/github.com/fluffle/goirc/client/doc.go @@ -0,0 +1,34 @@ +// Package client implements an IRC client. It handles protocol basics +// such as initial connection and responding to server PINGs, and has +// optional state tracking support which will keep tabs on every nick +// present in the same channels as the client. Other features include +// SSL support, automatic splitting of long lines, and panic recovery +// for handlers. +// +// Incoming IRC messages are parsed into client.Line structs and trigger +// events based on the IRC verb (e.g. PRIVMSG) of the message. Handlers +// for these events conform to the client.Handler interface; a HandlerFunc +// type to wrap bare functions is provided a-la the net/http package. +// +// Creating a client, adding a handler and connecting to a server looks +// soemthing like this, for the simple case: +// +// // Create a new client, which will connect with the nick "myNick" +// irc := client.SimpleClient("myNick") +// +// // Add a handler that waits for the "disconnected" event and +// // closes a channel to signal everything is done. +// disconnected := make(chan struct{}) +// c.HandleFunc("disconnected", func(c *client.Conn, l *client.Line) { +// close(disconnected) +// }) +// +// // Connect to an IRC server. +// if err := c.ConnectTo("irc.freenode.net"); err != nil { +// log.Fatalf("Connection error: %v\n", err) +// } +// +// // Wait for disconnection. +// <-disconnected +// +package client diff --git a/vendor/github.com/fluffle/goirc/client/handlers.go b/vendor/github.com/fluffle/goirc/client/handlers.go new file mode 100644 index 0000000..b538579 --- /dev/null +++ b/vendor/github.com/fluffle/goirc/client/handlers.go @@ -0,0 +1,105 @@ +package client + +// this file contains the basic set of event handlers +// to manage tracking an irc connection etc. + +import ( + "strings" + "time" +) + +// sets up the internal event handlers to do essential IRC protocol things +var intHandlers = map[string]HandlerFunc{ + REGISTER: (*Conn).h_REGISTER, + "001": (*Conn).h_001, + "433": (*Conn).h_433, + CTCP: (*Conn).h_CTCP, + NICK: (*Conn).h_NICK, + PING: (*Conn).h_PING, +} + +func (conn *Conn) addIntHandlers() { + for n, h := range intHandlers { + // internal handlers are essential for the IRC client + // to function, so we don't save their Removers here + conn.handle(n, h) + } +} + +// Basic ping/pong handler +func (conn *Conn) h_PING(line *Line) { + conn.Pong(line.Args[0]) +} + +// Handler for initial registration with server once tcp connection is made. +func (conn *Conn) h_REGISTER(line *Line) { + if conn.cfg.Pass != "" { + conn.Pass(conn.cfg.Pass) + } + conn.Nick(conn.cfg.Me.Nick) + conn.User(conn.cfg.Me.Ident, conn.cfg.Me.Name) +} + +// Handler to trigger a CONNECTED event on receipt of numeric 001 +func (conn *Conn) h_001(line *Line) { + // we're connected! + conn.dispatch(&Line{Cmd: CONNECTED, Time: time.Now()}) + // and we're being given our hostname (from the server's perspective) + t := line.Args[len(line.Args)-1] + if idx := strings.LastIndex(t, " "); idx != -1 { + t = t[idx+1:] + if idx = strings.Index(t, "@"); idx != -1 { + if conn.st != nil { + me := conn.Me() + conn.st.NickInfo(me.Nick, me.Ident, t[idx+1:], me.Name) + } else { + conn.cfg.Me.Host = t[idx+1:] + } + } + } +} + +// XXX: do we need 005 protocol support message parsing here? +// probably in the future, but I can't quite be arsed yet. +/* + :irc.pl0rt.org 005 GoTest CMDS=KNOCK,MAP,DCCALLOW,USERIP UHNAMES NAMESX SAFELIST HCN MAXCHANNELS=20 CHANLIMIT=#:20 MAXLIST=b:60,e:60,I:60 NICKLEN=30 CHANNELLEN=32 TOPICLEN=307 KICKLEN=307 AWAYLEN=307 :are supported by this server + :irc.pl0rt.org 005 GoTest MAXTARGETS=20 WALLCHOPS WATCH=128 WATCHOPTS=A SILENCE=15 MODES=12 CHANTYPES=# PREFIX=(qaohv)~&@%+ CHANMODES=beI,kfL,lj,psmntirRcOAQKVCuzNSMT NETWORK=bb101.net CASEMAPPING=ascii EXTBAN=~,cqnr ELIST=MNUCT :are supported by this server + :irc.pl0rt.org 005 GoTest STATUSMSG=~&@%+ EXCEPTS INVEX :are supported by this server +*/ + +// Handler to deal with "433 :Nickname already in use" +func (conn *Conn) h_433(line *Line) { + // Args[1] is the new nick we were attempting to acquire + me := conn.Me() + neu := conn.cfg.NewNick(line.Args[1]) + conn.Nick(neu) + if !line.argslen(1) { + return + } + // if this is happening before we're properly connected (i.e. the nick + // we sent in the initial NICK command is in use) we will not receive + // a NICK message to confirm our change of nick, so ReNick here... + if line.Args[1] == me.Nick { + if conn.st != nil { + conn.cfg.Me = conn.st.ReNick(me.Nick, neu) + } else { + conn.cfg.Me.Nick = neu + } + } +} + +// Handle VERSION requests and CTCP PING +func (conn *Conn) h_CTCP(line *Line) { + if line.Args[0] == VERSION { + conn.CtcpReply(line.Nick, VERSION, conn.cfg.Version) + } else if line.Args[0] == PING && line.argslen(2) { + conn.CtcpReply(line.Nick, PING, line.Args[2]) + } +} + +// Handle updating our own NICK if we're not using the state tracker +func (conn *Conn) h_NICK(line *Line) { + if conn.st == nil && line.Nick == conn.cfg.Me.Nick { + conn.cfg.Me.Nick = line.Args[0] + } +} diff --git a/vendor/github.com/fluffle/goirc/client/handlers_test.go b/vendor/github.com/fluffle/goirc/client/handlers_test.go new file mode 100644 index 0000000..7808022 --- /dev/null +++ b/vendor/github.com/fluffle/goirc/client/handlers_test.go @@ -0,0 +1,451 @@ +package client + +import ( + "github.com/fluffle/goirc/state" + "github.com/golang/mock/gomock" + "testing" + "time" +) + +// This test performs a simple end-to-end verification of correct line parsing +// and event dispatch as well as testing the PING handler. All the other tests +// in this file will call their respective handlers synchronously, otherwise +// testing becomes more difficult. +func TestPING(t *testing.T) { + _, s := setUp(t) + defer s.tearDown() + s.nc.Send("PING :1234567890") + s.nc.Expect("PONG :1234567890") +} + +// Test the REGISTER handler matches section 3.1 of rfc2812 +func TestREGISTER(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + c.h_REGISTER(&Line{Cmd: REGISTER}) + s.nc.Expect("NICK test") + s.nc.Expect("USER test 12 * :Testing IRC") + s.nc.ExpectNothing() + + c.cfg.Pass = "12345" + c.cfg.Me.Ident = "idiot" + c.cfg.Me.Name = "I've got the same combination on my luggage!" + c.h_REGISTER(&Line{Cmd: REGISTER}) + s.nc.Expect("PASS 12345") + s.nc.Expect("NICK test") + s.nc.Expect("USER idiot 12 * :I've got the same combination on my luggage!") + s.nc.ExpectNothing() +} + +// Test the handler for 001 / RPL_WELCOME +func Test001(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + l := ParseLine(":irc.server.org 001 test :Welcome to IRC test!ident@somehost.com") + // Set up a handler to detect whether connected handler is called from 001 + hcon := false + c.HandleFunc("connected", func(conn *Conn, line *Line) { + hcon = true + }) + + // Test state tracking first. + gomock.InOrder( + s.st.EXPECT().Me().Return(c.cfg.Me), + s.st.EXPECT().NickInfo("test", "test", "somehost.com", "Testing IRC"), + ) + // Call handler with a valid 001 line + c.h_001(l) + <-time.After(time.Millisecond) + if !hcon { + t.Errorf("001 handler did not dispatch connected event.") + } + + // Now without state tracking. + c.st = nil + c.h_001(l) + // Check host parsed correctly + if c.cfg.Me.Host != "somehost.com" { + t.Errorf("Host parsing failed, host is '%s'.", c.cfg.Me.Host) + } + c.st = s.st +} + +// Test the handler for 433 / ERR_NICKNAMEINUSE +func Test433(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + // Call handler with a 433 line, not triggering c.cfg.Me.Renick() + s.st.EXPECT().Me().Return(c.cfg.Me) + c.h_433(ParseLine(":irc.server.org 433 test new :Nickname is already in use.")) + s.nc.Expect("NICK new_") + + // Send a line that will trigger a renick. This happens when our wanted + // nick is unavailable during initial negotiation, so we must choose a + // different one before the connection can proceed. No NICK line will be + // sent by the server to confirm nick change in this case. + gomock.InOrder( + s.st.EXPECT().Me().Return(c.cfg.Me), + s.st.EXPECT().ReNick("test", "test_").Return(c.cfg.Me), + ) + c.h_433(ParseLine(":irc.server.org 433 test test :Nickname is already in use.")) + s.nc.Expect("NICK test_") + + // Test the code path that *doesn't* involve state tracking. + c.st = nil + c.h_433(ParseLine(":irc.server.org 433 test test :Nickname is already in use.")) + s.nc.Expect("NICK test_") + + if c.cfg.Me.Nick != "test_" { + t.Errorf("My nick not updated from '%s'.", c.cfg.Me.Nick) + } + c.st = s.st +} + +// Test the handler for NICK messages when state tracking is disabled +func TestNICK(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + // State tracking is enabled by default in setUp + c.st = nil + + // Call handler with a NICK line changing "our" nick to test1. + c.h_NICK(ParseLine(":test!test@somehost.com NICK :test1")) + + // Verify that our Nick has changed + if c.cfg.Me.Nick != "test1" { + t.Errorf("NICK did not result in changing our nick.") + } + + // Send a NICK line for something that isn't us. + c.h_NICK(ParseLine(":blah!moo@cows.com NICK :milk")) + + // Verify that our Nick hasn't changed + if c.cfg.Me.Nick != "test1" { + t.Errorf("NICK did not result in changing our nick.") + } + + // Re-enable state tracking and send a line that *should* change nick. + c.st = s.st + c.h_NICK(ParseLine(":test1!test@somehost.com NICK :test2")) + + // Verify that our Nick hasn't changed (should be handled by h_STNICK). + if c.cfg.Me.Nick != "test1" { + t.Errorf("NICK changed our nick when state tracking enabled.") + } +} + +// Test the handler for CTCP messages +func TestCTCP(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + // Call handler with CTCP VERSION + c.h_CTCP(ParseLine(":blah!moo@cows.com PRIVMSG test :\001VERSION\001")) + + // Expect a version reply + s.nc.Expect("NOTICE blah :\001VERSION Powered by GoIRC\001") + + // Call handler with CTCP PING + c.h_CTCP(ParseLine(":blah!moo@cows.com PRIVMSG test :\001PING 1234567890\001")) + + // Expect a ping reply + s.nc.Expect("NOTICE blah :\001PING 1234567890\001") + + // Call handler with CTCP UNKNOWN + c.h_CTCP(ParseLine(":blah!moo@cows.com PRIVMSG test :\001UNKNOWN ctcp\001")) +} + +// Test the handler for JOIN messages +func TestJOIN(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + // The state tracker should be creating a new channel in this first test + chan1 := &state.Channel{Name: "#test1"} + + gomock.InOrder( + s.st.EXPECT().GetChannel("#test1").Return(nil), + s.st.EXPECT().GetNick("test").Return(c.cfg.Me), + s.st.EXPECT().Me().Return(c.cfg.Me), + s.st.EXPECT().NewChannel("#test1").Return(chan1), + s.st.EXPECT().Associate("#test1", "test"), + ) + + // Use #test1 to test expected behaviour + // Call handler with JOIN by test to #test1 + c.h_JOIN(ParseLine(":test!test@somehost.com JOIN :#test1")) + + // Verify that the MODE and WHO commands are sent correctly + s.nc.Expect("MODE #test1") + s.nc.Expect("WHO #test1") + + // In this second test, we should be creating a new nick + nick1 := &state.Nick{Nick: "user1"} + + gomock.InOrder( + s.st.EXPECT().GetChannel("#test1").Return(chan1), + s.st.EXPECT().GetNick("user1").Return(nil), + s.st.EXPECT().NewNick("user1").Return(nick1), + s.st.EXPECT().NickInfo("user1", "ident1", "host1.com", "").Return(nick1), + s.st.EXPECT().Associate("#test1", "user1"), + ) + + // OK, now #test1 exists, JOIN another user we don't know about + c.h_JOIN(ParseLine(":user1!ident1@host1.com JOIN :#test1")) + + // Verify that the WHO command is sent correctly + s.nc.Expect("WHO user1") + + // In this third test, we'll be pretending we know about the nick already. + nick2 := &state.Nick{Nick: "user2"} + gomock.InOrder( + s.st.EXPECT().GetChannel("#test1").Return(chan1), + s.st.EXPECT().GetNick("user2").Return(nick2), + s.st.EXPECT().Associate("#test1", "user2"), + ) + c.h_JOIN(ParseLine(":user2!ident2@host2.com JOIN :#test1")) + + // Test error paths + gomock.InOrder( + // unknown channel, unknown nick + s.st.EXPECT().GetChannel("#test2").Return(nil), + s.st.EXPECT().GetNick("blah").Return(nil), + s.st.EXPECT().Me().Return(c.cfg.Me), + // unknown channel, known nick that isn't Me. + s.st.EXPECT().GetChannel("#test2").Return(nil), + s.st.EXPECT().GetNick("user2").Return(nick2), + s.st.EXPECT().Me().Return(c.cfg.Me), + ) + c.h_JOIN(ParseLine(":blah!moo@cows.com JOIN :#test2")) + c.h_JOIN(ParseLine(":user2!ident2@host2.com JOIN :#test2")) +} + +// Test the handler for PART messages +func TestPART(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + // PART should dissociate a nick from a channel. + s.st.EXPECT().Dissociate("#test1", "user1") + c.h_PART(ParseLine(":user1!ident1@host1.com PART #test1 :Bye!")) +} + +// Test the handler for KICK messages +// (this is very similar to the PART message test) +func TestKICK(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + // KICK should dissociate a nick from a channel. + s.st.EXPECT().Dissociate("#test1", "user1") + c.h_KICK(ParseLine(":test!test@somehost.com KICK #test1 user1 :Bye!")) +} + +// Test the handler for QUIT messages +func TestQUIT(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + // Have user1 QUIT. All possible errors handled by state tracker \o/ + s.st.EXPECT().DelNick("user1") + c.h_QUIT(ParseLine(":user1!ident1@host1.com QUIT :Bye!")) +} + +// Test the handler for MODE messages +func TestMODE(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + // Channel modes + gomock.InOrder( + s.st.EXPECT().GetChannel("#test1").Return(&state.Channel{Name: "#test1"}), + s.st.EXPECT().ChannelModes("#test1", "+sk", "somekey"), + ) + c.h_MODE(ParseLine(":user1!ident1@host1.com MODE #test1 +sk somekey")) + + // Nick modes for Me. + gomock.InOrder( + s.st.EXPECT().GetChannel("test").Return(nil), + s.st.EXPECT().GetNick("test").Return(c.cfg.Me), + s.st.EXPECT().Me().Return(c.cfg.Me), + s.st.EXPECT().NickModes("test", "+i"), + ) + c.h_MODE(ParseLine(":test!test@somehost.com MODE test +i")) + + // Check error paths + gomock.InOrder( + // send a valid user mode that's not us + s.st.EXPECT().GetChannel("user1").Return(nil), + s.st.EXPECT().GetNick("user1").Return(&state.Nick{Nick: "user1"}), + s.st.EXPECT().Me().Return(c.cfg.Me), + // Send a random mode for an unknown channel + s.st.EXPECT().GetChannel("#test2").Return(nil), + s.st.EXPECT().GetNick("#test2").Return(nil), + ) + c.h_MODE(ParseLine(":user1!ident1@host1.com MODE user1 +w")) + c.h_MODE(ParseLine(":user1!ident1@host1.com MODE #test2 +is")) +} + +// Test the handler for TOPIC messages +func TestTOPIC(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + // Ensure TOPIC reply calls Topic + gomock.InOrder( + s.st.EXPECT().GetChannel("#test1").Return(&state.Channel{Name: "#test1"}), + s.st.EXPECT().Topic("#test1", "something something"), + ) + c.h_TOPIC(ParseLine(":user1!ident1@host1.com TOPIC #test1 :something something")) + + // Check error paths -- send a topic for an unknown channel + s.st.EXPECT().GetChannel("#test2").Return(nil) + c.h_TOPIC(ParseLine(":user1!ident1@host1.com TOPIC #test2 :dark side")) +} + +// Test the handler for 311 / RPL_WHOISUSER +func Test311(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + // Ensure 311 reply calls NickInfo + gomock.InOrder( + s.st.EXPECT().GetNick("user1").Return(&state.Nick{Nick: "user1"}), + s.st.EXPECT().Me().Return(c.cfg.Me), + s.st.EXPECT().NickInfo("user1", "ident1", "host1.com", "name"), + ) + c.h_311(ParseLine(":irc.server.org 311 test user1 ident1 host1.com * :name")) + + // Check error paths -- send a 311 for an unknown nick + s.st.EXPECT().GetNick("user2").Return(nil) + c.h_311(ParseLine(":irc.server.org 311 test user2 ident2 host2.com * :dongs")) +} + +// Test the handler for 324 / RPL_CHANNELMODEIS +func Test324(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + // Ensure 324 reply calls ChannelModes + gomock.InOrder( + s.st.EXPECT().GetChannel("#test1").Return(&state.Channel{Name: "#test1"}), + s.st.EXPECT().ChannelModes("#test1", "+sk", "somekey"), + ) + c.h_324(ParseLine(":irc.server.org 324 test #test1 +sk somekey")) + + // Check error paths -- send 324 for an unknown channel + s.st.EXPECT().GetChannel("#test2").Return(nil) + c.h_324(ParseLine(":irc.server.org 324 test #test2 +pmt")) +} + +// Test the handler for 332 / RPL_TOPIC +func Test332(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + // Ensure 332 reply calls Topic + gomock.InOrder( + s.st.EXPECT().GetChannel("#test1").Return(&state.Channel{Name: "#test1"}), + s.st.EXPECT().Topic("#test1", "something something"), + ) + c.h_332(ParseLine(":irc.server.org 332 test #test1 :something something")) + + // Check error paths -- send 332 for an unknown channel + s.st.EXPECT().GetChannel("#test2").Return(nil) + c.h_332(ParseLine(":irc.server.org 332 test #test2 :dark side")) +} + +// Test the handler for 352 / RPL_WHOREPLY +func Test352(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + // Ensure 352 reply calls NickInfo and NickModes + gomock.InOrder( + s.st.EXPECT().GetNick("user1").Return(&state.Nick{Nick: "user1"}), + s.st.EXPECT().Me().Return(c.cfg.Me), + s.st.EXPECT().NickInfo("user1", "ident1", "host1.com", "name"), + ) + c.h_352(ParseLine(":irc.server.org 352 test #test1 ident1 host1.com irc.server.org user1 G :0 name")) + + // Check that modes are set correctly from WHOREPLY + gomock.InOrder( + s.st.EXPECT().GetNick("user1").Return(&state.Nick{Nick: "user1"}), + s.st.EXPECT().Me().Return(c.cfg.Me), + s.st.EXPECT().NickInfo("user1", "ident1", "host1.com", "name"), + s.st.EXPECT().NickModes("user1", "+o"), + s.st.EXPECT().NickModes("user1", "+i"), + ) + c.h_352(ParseLine(":irc.server.org 352 test #test1 ident1 host1.com irc.server.org user1 H* :0 name")) + + // Check error paths -- send a 352 for an unknown nick + s.st.EXPECT().GetNick("user2").Return(nil) + c.h_352(ParseLine(":irc.server.org 352 test #test2 ident2 host2.com irc.server.org user2 G :0 fooo")) +} + +// Test the handler for 353 / RPL_NAMREPLY +func Test353(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + // 353 handler is called twice, so GetChannel will be called twice + s.st.EXPECT().GetChannel("#test1").Return(&state.Channel{Name: "#test1"}).Times(2) + gomock.InOrder( + // "test" is Me, i am known, and already on the channel + s.st.EXPECT().GetNick("test").Return(c.cfg.Me), + s.st.EXPECT().IsOn("#test1", "test").Return(&state.ChanPrivs{}, true), + // user1 is known, but not on the channel, so should be associated + s.st.EXPECT().GetNick("user1").Return(&state.Nick{Nick: "user1"}), + s.st.EXPECT().IsOn("#test1", "user1").Return(nil, false), + s.st.EXPECT().Associate("#test1", "user1").Return(&state.ChanPrivs{}), + s.st.EXPECT().ChannelModes("#test1", "+o", "user1"), + ) + for n, m := range map[string]string{ + "user2": "", + "voice": "+v", + "halfop": "+h", + "op": "+o", + "admin": "+a", + "owner": "+q", + } { + calls := []*gomock.Call{ + s.st.EXPECT().GetNick(n).Return(nil), + s.st.EXPECT().NewNick(n).Return(&state.Nick{Nick: n}), + s.st.EXPECT().IsOn("#test1", n).Return(nil, false), + s.st.EXPECT().Associate("#test1", n).Return(&state.ChanPrivs{}), + } + if m != "" { + calls = append(calls, s.st.EXPECT().ChannelModes("#test1", m, n)) + } + gomock.InOrder(calls...) + } + + // Send a couple of names replies (complete with trailing space) + c.h_353(ParseLine(":irc.server.org 353 test = #test1 :test @user1 user2 +voice ")) + c.h_353(ParseLine(":irc.server.org 353 test = #test1 :%halfop @op &admin ~owner ")) + + // Check error paths -- send 353 for an unknown channel + s.st.EXPECT().GetChannel("#test2").Return(nil) + c.h_353(ParseLine(":irc.server.org 353 test = #test2 :test ~user3")) +} + +// Test the handler for 671 (unreal specific) +func Test671(t *testing.T) { + c, s := setUp(t) + defer s.tearDown() + + // Ensure 671 reply calls NickModes + gomock.InOrder( + s.st.EXPECT().GetNick("user1").Return(&state.Nick{Nick: "user1"}), + s.st.EXPECT().NickModes("user1", "+z"), + ) + c.h_671(ParseLine(":irc.server.org 671 test user1 :some ignored text")) + + // Check error paths -- send a 671 for an unknown nick + s.st.EXPECT().GetNick("user2").Return(nil) + c.h_671(ParseLine(":irc.server.org 671 test user2 :some ignored text")) +} diff --git a/vendor/github.com/fluffle/goirc/client/line.go b/vendor/github.com/fluffle/goirc/client/line.go new file mode 100644 index 0000000..bfa473a --- /dev/null +++ b/vendor/github.com/fluffle/goirc/client/line.go @@ -0,0 +1,216 @@ +package client + +import ( + "runtime" + "strings" + "time" + + "github.com/fluffle/goirc/logging" +) + +var tagsReplacer = strings.NewReplacer("\\:", ";", "\\s", " ", "\\r", "\r", "\\n", "\n") + +// We parse an incoming line into this struct. Line.Cmd is used as the trigger +// name for incoming event handlers and is the IRC verb, the first sequence +// of non-whitespace characters after ":nick!user@host", e.g. PRIVMSG. +// Raw =~ ":nick!user@host cmd args[] :text" +// Src == "nick!user@host" +// Cmd == e.g. PRIVMSG, 332 +type Line struct { + Tags map[string]string + Nick, Ident, Host, Src string + Cmd, Raw string + Args []string + Time time.Time +} + +// Copy returns a deep copy of the Line. +func (l *Line) Copy() *Line { + nl := *l + nl.Args = make([]string, len(l.Args)) + copy(nl.Args, l.Args) + if l.Tags != nil { + nl.Tags = make(map[string]string) + for k, v := range l.Tags { + nl.Tags[k] = v + } + } + return &nl +} + +// Text returns the contents of the text portion of a line. This only really +// makes sense for lines with a :text part, but there are a lot of them. +func (line *Line) Text() string { + if len(line.Args) > 0 { + return line.Args[len(line.Args)-1] + } + return "" +} + +// Target returns the contextual target of the line, usually the first Arg +// for the IRC verb. If the line was broadcast from a channel, the target +// will be that channel. If the line was sent directly by a user, the target +// will be that user. +func (line *Line) Target() string { + // TODO(fluffle): Add 005 CHANTYPES parsing for this? + switch line.Cmd { + case PRIVMSG, NOTICE, ACTION: + if !line.Public() { + return line.Nick + } + case CTCP, CTCPREPLY: + if !line.Public() { + return line.Nick + } + return line.Args[1] + } + if len(line.Args) > 0 { + return line.Args[0] + } + return "" +} + +// Public returns true if the line is the result of an IRC user sending +// a message to a channel the client has joined instead of directly +// to the client. +// +// NOTE: This is very permissive, allowing all 4 RFC channel types even if +// your server doesn't technically support them. +func (line *Line) Public() bool { + switch line.Cmd { + case PRIVMSG, NOTICE, ACTION: + switch line.Args[0][0] { + case '#', '&', '+', '!': + return true + } + case CTCP, CTCPREPLY: + // CTCP prepends the CTCP verb to line.Args, thus for the message + // :nick!user@host PRIVMSG #foo :\001BAR baz\001 + // line.Args contains: []string{"BAR", "#foo", "baz"} + // TODO(fluffle): Arguably this is broken, and we should have + // line.Args containing: []string{"#foo", "BAR", "baz"} + // ... OR change conn.Ctcp()'s argument order to be consistent. + switch line.Args[1][0] { + case '#', '&', '+', '!': + return true + } + } + return false +} + +// ParseLine creates a Line from an incoming message from the IRC server. +// +// It contains special casing for CTCP messages, most notably CTCP ACTION. +// All CTCP messages have the \001 bytes stripped from the message and the +// CTCP command separated from any subsequent text. Then, CTCP ACTIONs are +// rewritten such that Line.Cmd == ACTION. Other CTCP messages have Cmd +// set to CTCP or CTCPREPLY, and the CTCP command prepended to line.Args. +// +// ParseLine also parses IRCv3 tags, if received. If a line does not have +// the tags section, Line.Tags will be nil. Tags are optional, and will +// only be included after the correct CAP command. +// +// http://ircv3.net/specs/core/capability-negotiation-3.1.html +// http://ircv3.net/specs/core/message-tags-3.2.html +func ParseLine(s string) *Line { + line := &Line{Raw: s} + + if s == "" { + return nil + } + + if s[0] == '@' { + var rawTags string + line.Tags = make(map[string]string) + if idx := strings.Index(s, " "); idx != -1 { + rawTags, s = s[1:idx], s[idx+1:] + } else { + return nil + } + + // ; is represented as \: in a tag, so it's safe to split on ; + for _, tag := range strings.Split(rawTags, ";") { + if tag == "" { + continue + } + + pair := strings.SplitN(tagsReplacer.Replace(tag), "=", 2) + if len(pair) < 2 { + line.Tags[tag] = "" + } else { + line.Tags[pair[0]] = pair[1] + } + } + } + + if s[0] == ':' { + // remove a source and parse it + if idx := strings.Index(s, " "); idx != -1 { + line.Src, s = s[1:idx], s[idx+1:] + } else { + // pretty sure we shouldn't get here ... + return nil + } + + // src can be the hostname of the irc server or a nick!user@host + line.Host = line.Src + nidx, uidx := strings.Index(line.Src, "!"), strings.Index(line.Src, "@") + if uidx != -1 && nidx != -1 { + line.Nick = line.Src[:nidx] + line.Ident = line.Src[nidx+1 : uidx] + line.Host = line.Src[uidx+1:] + } + } + + // now we're here, we've parsed a :nick!user@host or :server off + // s should contain "cmd args[] :text" + args := strings.SplitN(s, " :", 2) + if len(args) > 1 { + args = append(strings.Fields(args[0]), args[1]) + } else { + args = strings.Fields(args[0]) + } + line.Cmd = strings.ToUpper(args[0]) + if len(args) > 1 { + line.Args = args[1:] + } + + // So, I think CTCP and (in particular) CTCP ACTION are better handled as + // separate events as opposed to forcing people to have gargantuan + // handlers to cope with the possibilities. + if (line.Cmd == PRIVMSG || line.Cmd == NOTICE) && + len(line.Args[1]) > 2 && + strings.HasPrefix(line.Args[1], "\001") && + strings.HasSuffix(line.Args[1], "\001") { + // WOO, it's a CTCP message + t := strings.SplitN(strings.Trim(line.Args[1], "\001"), " ", 2) + if len(t) > 1 { + // Replace the line with the unwrapped CTCP + line.Args[1] = t[1] + } + if c := strings.ToUpper(t[0]); c == ACTION && line.Cmd == PRIVMSG { + // make a CTCP ACTION it's own event a-la PRIVMSG + line.Cmd = c + } else { + // otherwise, dispatch a generic CTCP/CTCPREPLY event that + // contains the type of CTCP in line.Args[0] + if line.Cmd == PRIVMSG { + line.Cmd = CTCP + } else { + line.Cmd = CTCPREPLY + } + line.Args = append([]string{c}, line.Args...) + } + } + return line +} + +func (line *Line) argslen(minlen int) bool { + pc, _, _, _ := runtime.Caller(1) + fn := runtime.FuncForPC(pc) + if len(line.Args) <= minlen { + logging.Warn("%s: too few arguments: %s", fn.Name(), strings.Join(line.Args, " ")) + return false + } + return true +} diff --git a/vendor/github.com/fluffle/goirc/client/line_test.go b/vendor/github.com/fluffle/goirc/client/line_test.go new file mode 100644 index 0000000..88b758d --- /dev/null +++ b/vendor/github.com/fluffle/goirc/client/line_test.go @@ -0,0 +1,186 @@ +package client + +import ( + "reflect" + "testing" + "time" +) + +func TestLineCopy(t *testing.T) { + l1 := &Line{ + Tags: map[string]string{"foo": "bar", "fizz": "buzz"}, + Nick: "nick", + Ident: "ident", + Host: "host", + Src: "src", + Cmd: "cmd", + Raw: "raw", + Args: []string{"arg", "text"}, + Time: time.Now(), + } + + l2 := l1.Copy() + + // Ugly. Couldn't be bothered to bust out reflect and actually think. + if l2.Tags == nil || l2.Tags["foo"] != "bar" || l2.Tags["fizz"] != "buzz" || + l2.Nick != "nick" || l2.Ident != "ident" || l2.Host != "host" || + l2.Src != "src" || l2.Cmd != "cmd" || l2.Raw != "raw" || + l2.Args[0] != "arg" || l2.Args[1] != "text" || l2.Time != l1.Time { + t.Errorf("Line not copied correctly") + t.Errorf("l1: %#v\nl2: %#v", l1, l2) + } + + // Now, modify l2 and verify l1 not changed + l2.Tags["foo"] = "baz" + l2.Nick = l2.Nick[1:] + l2.Ident = "foo" + l2.Host = "" + l2.Args[0] = l2.Args[0][1:] + l2.Args[1] = "bar" + l2.Time = time.Now() + + if l2.Tags == nil || l2.Tags["foo"] != "baz" || l2.Tags["fizz"] != "buzz" || + l1.Nick != "nick" || l1.Ident != "ident" || l1.Host != "host" || + l1.Src != "src" || l1.Cmd != "cmd" || l1.Raw != "raw" || + l1.Args[0] != "arg" || l1.Args[1] != "text" || l1.Time == l2.Time { + t.Errorf("Original modified when copy changed") + t.Errorf("l1: %#v\nl2: %#v", l1, l2) + } +} + +func TestLineText(t *testing.T) { + tests := []struct { + in *Line + out string + }{ + {&Line{}, ""}, + {&Line{Args: []string{"one thing"}}, "one thing"}, + {&Line{Args: []string{"one", "two"}}, "two"}, + } + + for i, test := range tests { + out := test.in.Text() + if out != test.out { + t.Errorf("test %d: expected: '%s', got '%s'", i, test.out, out) + } + } +} + +func TestLineTarget(t *testing.T) { + tests := []struct { + in *Line + out string + }{ + {&Line{}, ""}, + {&Line{Cmd: JOIN, Args: []string{"#foo"}}, "#foo"}, + {&Line{Cmd: PART, Args: []string{"#foo", "bye"}}, "#foo"}, + {&Line{Cmd: PRIVMSG, Args: []string{"Me", "la"}, Nick: "Them"}, "Them"}, + {&Line{Cmd: NOTICE, Args: []string{"Me", "la"}, Nick: "Them"}, "Them"}, + {&Line{Cmd: ACTION, Args: []string{"Me", "la"}, Nick: "Them"}, "Them"}, + {&Line{Cmd: CTCP, Args: []string{"PING", "Me", "1"}, Nick: "Them"}, "Them"}, + {&Line{Cmd: CTCPREPLY, Args: []string{"PONG", "Me", "2"}, Nick: "Them"}, "Them"}, + {&Line{Cmd: PRIVMSG, Args: []string{"#foo", "la"}, Nick: "Them"}, "#foo"}, + {&Line{Cmd: NOTICE, Args: []string{"&foo", "la"}, Nick: "Them"}, "&foo"}, + {&Line{Cmd: ACTION, Args: []string{"!foo", "la"}, Nick: "Them"}, "!foo"}, + {&Line{Cmd: CTCP, Args: []string{"PING", "#foo", "1"}, Nick: "Them"}, "#foo"}, + {&Line{Cmd: CTCPREPLY, Args: []string{"PONG", "#foo", "2"}, Nick: "Them"}, "#foo"}, + } + + for i, test := range tests { + out := test.in.Target() + if out != test.out { + t.Errorf("test %d: expected: '%s', got '%s'", i, test.out, out) + } + } +} + +func TestLineTags(t *testing.T) { + tests := []struct { + in string + out *Line + }{ + { // Make sure ERROR lines work + "ERROR :Closing Link: example.org (Too many user connections (global))", + &Line{ + Nick: "", + Ident: "", + Host: "", + Src: "", + Cmd: ERROR, + Raw: "ERROR :Closing Link: example.org (Too many user connections (global))", + Args: []string{"Closing Link: example.org (Too many user connections (global))"}, + }, + }, + { // Make sure non-tagged lines work + ":nick!ident@host.com PRIVMSG me :Hello", + &Line{ + Nick: "nick", + Ident: "ident", + Host: "host.com", + Src: "nick!ident@host.com", + Cmd: PRIVMSG, + Raw: ":nick!ident@host.com PRIVMSG me :Hello", + Args: []string{"me", "Hello"}, + }, + }, + { // Tags example from the spec + "@aaa=bbb;ccc;example.com/ddd=eee :nick!ident@host.com PRIVMSG me :Hello", + &Line{ + Tags: map[string]string{"aaa": "bbb", "ccc": "", "example.com/ddd": "eee"}, + Nick: "nick", + Ident: "ident", + Host: "host.com", + Src: "nick!ident@host.com", + Cmd: PRIVMSG, + Raw: "@aaa=bbb;ccc;example.com/ddd=eee :nick!ident@host.com PRIVMSG me :Hello", + Args: []string{"me", "Hello"}, + }, + }, + { // Test escaped characters + "@\\:=\\:;\\s=\\s;\\r=\\r;\\n=\\n :nick!ident@host.com PRIVMSG me :Hello", + &Line{ + Tags: map[string]string{";": ";", " ": " ", "\r": "\r", "\n": "\n"}, + Nick: "nick", + Ident: "ident", + Host: "host.com", + Src: "nick!ident@host.com", + Cmd: PRIVMSG, + Raw: "@\\:=\\:;\\s=\\s;\\r=\\r;\\n=\\n :nick!ident@host.com PRIVMSG me :Hello", + Args: []string{"me", "Hello"}, + }, + }, + { // Skip empty tag + "@a=a; :nick!ident@host.com PRIVMSG me :Hello", + &Line{ + Tags: map[string]string{"a": "a"}, + Nick: "nick", + Ident: "ident", + Host: "host.com", + Src: "nick!ident@host.com", + Cmd: PRIVMSG, + Raw: "@a=a; :nick!ident@host.com PRIVMSG me :Hello", + Args: []string{"me", "Hello"}, + }, + }, + { // = in tag + "@a=a=a; :nick!ident@host.com PRIVMSG me :Hello", + &Line{ + Tags: map[string]string{"a": "a=a"}, + Nick: "nick", + Ident: "ident", + Host: "host.com", + Src: "nick!ident@host.com", + Cmd: PRIVMSG, + Raw: "@a=a=a; :nick!ident@host.com PRIVMSG me :Hello", + Args: []string{"me", "Hello"}, + }, + }, + } + + for i, test := range tests { + got := ParseLine(test.in) + if !reflect.DeepEqual(got, test.out) { + t.Errorf("test %d:\nexpected %#v\ngot %#v", i, test.out, got) + } + } +} diff --git a/vendor/github.com/fluffle/goirc/client/mocknetconn_test.go b/vendor/github.com/fluffle/goirc/client/mocknetconn_test.go new file mode 100644 index 0000000..e736c88 --- /dev/null +++ b/vendor/github.com/fluffle/goirc/client/mocknetconn_test.go @@ -0,0 +1,154 @@ +package client + +import ( + "io" + "net" + "os" + "strings" + "testing" + "time" +) + +type mockNetConn struct { + *testing.T + + In, Out chan string + in, out chan []byte + die chan struct{} + + closed bool + rt, wt time.Time +} + +func MockNetConn(t *testing.T) *mockNetConn { + // Our mock connection is a testing object + m := &mockNetConn{T: t, die: make(chan struct{})} + + // buffer input + m.In = make(chan string, 20) + m.in = make(chan []byte) + go func() { + for { + select { + case <-m.die: + return + case s := <-m.In: + m.in <- []byte(s) + } + } + }() + + // buffer output + m.Out = make(chan string) + m.out = make(chan []byte, 20) + go func() { + for { + select { + case <-m.die: + return + case b := <-m.out: + m.Out <- string(b) + } + } + }() + + return m +} + +// Test helpers +func (m *mockNetConn) Send(s string) { + m.In <- s + "\r\n" +} + +func (m *mockNetConn) Expect(e string) { + select { + case <-time.After(time.Millisecond): + m.Errorf("Mock connection did not receive expected output.\n\t"+ + "Expected: '%s', got nothing.", e) + case s := <-m.Out: + s = strings.Trim(s, "\r\n") + if e != s { + m.Errorf("Mock connection received unexpected value.\n\t"+ + "Expected: '%s'\n\tGot: '%s'", e, s) + } + } +} + +func (m *mockNetConn) ExpectNothing() { + select { + case <-time.After(time.Millisecond): + case s := <-m.Out: + s = strings.Trim(s, "\r\n") + m.Errorf("Mock connection received unexpected output.\n\t"+ + "Expected nothing, got: '%s'", s) + } +} + +// Implement net.Conn interface +func (m *mockNetConn) Read(b []byte) (int, error) { + if m.Closed() { + return 0, os.ErrInvalid + } + l := 0 + select { + case s := <-m.in: + l = len(s) + copy(b, s) + case <-m.die: + return 0, io.EOF + } + return l, nil +} + +func (m *mockNetConn) Write(s []byte) (int, error) { + if m.Closed() { + return 0, os.ErrInvalid + } + b := make([]byte, len(s)) + copy(b, s) + m.out <- b + return len(s), nil +} + +func (m *mockNetConn) Close() error { + if m.Closed() { + return os.ErrInvalid + } + // Shut down *ALL* the goroutines! + // This will trigger an EOF event in Read() too + close(m.die) + return nil +} + +func (m *mockNetConn) Closed() bool { + select { + case <-m.die: + return true + default: + return false + } +} + +func (m *mockNetConn) LocalAddr() net.Addr { + return &net.IPAddr{IP: net.IPv4(127, 0, 0, 1)} +} + +func (m *mockNetConn) RemoteAddr() net.Addr { + return &net.IPAddr{IP: net.IPv4(127, 0, 0, 1)} +} + +func (m *mockNetConn) SetDeadline(t time.Time) error { + m.rt = t + m.wt = t + return nil +} + +func (m *mockNetConn) SetReadDeadline(t time.Time) error { + m.rt = t + return nil +} + +func (m *mockNetConn) SetWriteDeadline(t time.Time) error { + m.wt = t + return nil +} diff --git a/vendor/github.com/fluffle/goirc/client/state_handlers.go b/vendor/github.com/fluffle/goirc/client/state_handlers.go new file mode 100644 index 0000000..847679c --- /dev/null +++ b/vendor/github.com/fluffle/goirc/client/state_handlers.go @@ -0,0 +1,262 @@ +package client + +// this file contains the extra set of event handlers +// to manage tracking state for an IRC connection + +import ( + "strings" + + "github.com/fluffle/goirc/logging" +) + +var stHandlers = map[string]HandlerFunc{ + "JOIN": (*Conn).h_JOIN, + "KICK": (*Conn).h_KICK, + "MODE": (*Conn).h_MODE, + "NICK": (*Conn).h_STNICK, + "PART": (*Conn).h_PART, + "QUIT": (*Conn).h_QUIT, + "TOPIC": (*Conn).h_TOPIC, + "311": (*Conn).h_311, + "324": (*Conn).h_324, + "332": (*Conn).h_332, + "352": (*Conn).h_352, + "353": (*Conn).h_353, + "671": (*Conn).h_671, +} + +func (conn *Conn) addSTHandlers() { + for n, h := range stHandlers { + conn.stRemovers = append(conn.stRemovers, conn.handle(n, h)) + } +} + +func (conn *Conn) delSTHandlers() { + for _, h := range conn.stRemovers { + h.Remove() + } + conn.stRemovers = conn.stRemovers[:0] +} + +// Handle NICK messages that need to update the state tracker +func (conn *Conn) h_STNICK(line *Line) { + // all nicks should be handled the same way, our own included + conn.st.ReNick(line.Nick, line.Args[0]) +} + +// Handle JOINs to channels to maintain state +func (conn *Conn) h_JOIN(line *Line) { + ch := conn.st.GetChannel(line.Args[0]) + nk := conn.st.GetNick(line.Nick) + if ch == nil { + // first we've seen of this channel, so should be us joining it + // NOTE this will also take care of nk == nil && ch == nil + if !conn.Me().Equals(nk) { + logging.Warn("irc.JOIN(): JOIN to unknown channel %s received "+ + "from (non-me) nick %s", line.Args[0], line.Nick) + return + } + conn.st.NewChannel(line.Args[0]) + // since we don't know much about this channel, ask server for info + // we get the channel users automatically in 353 and the channel + // topic in 332 on join, so we just need to get the modes + conn.Mode(line.Args[0]) + // sending a WHO for the channel is MUCH more efficient than + // triggering a WHOIS on every nick from the 353 handler + conn.Who(line.Args[0]) + } + if nk == nil { + // this is the first we've seen of this nick + conn.st.NewNick(line.Nick) + conn.st.NickInfo(line.Nick, line.Ident, line.Host, "") + // since we don't know much about this nick, ask server for info + conn.Who(line.Nick) + } + // this takes care of both nick and channel linking \o/ + conn.st.Associate(line.Args[0], line.Nick) +} + +// Handle PARTs from channels to maintain state +func (conn *Conn) h_PART(line *Line) { + conn.st.Dissociate(line.Args[0], line.Nick) +} + +// Handle KICKs from channels to maintain state +func (conn *Conn) h_KICK(line *Line) { + if !line.argslen(1) { + return + } + // XXX: this won't handle autorejoining channels on KICK + // it's trivial to do this in a seperate handler... + conn.st.Dissociate(line.Args[0], line.Args[1]) +} + +// Handle other people's QUITs +func (conn *Conn) h_QUIT(line *Line) { + conn.st.DelNick(line.Nick) +} + +// Handle MODE changes for channels we know about (and our nick personally) +func (conn *Conn) h_MODE(line *Line) { + if !line.argslen(1) { + return + } + if ch := conn.st.GetChannel(line.Args[0]); ch != nil { + // channel modes first + conn.st.ChannelModes(line.Args[0], line.Args[1], line.Args[2:]...) + } else if nk := conn.st.GetNick(line.Args[0]); nk != nil { + // nick mode change, should be us + if !conn.Me().Equals(nk) { + logging.Warn("irc.MODE(): recieved MODE %s for (non-me) nick %s", + line.Args[1], line.Args[0]) + return + } + conn.st.NickModes(line.Args[0], line.Args[1]) + } else { + logging.Warn("irc.MODE(): not sure what to do with MODE %s", + strings.Join(line.Args, " ")) + } +} + +// Handle TOPIC changes for channels +func (conn *Conn) h_TOPIC(line *Line) { + if !line.argslen(1) { + return + } + if ch := conn.st.GetChannel(line.Args[0]); ch != nil { + conn.st.Topic(line.Args[0], line.Args[1]) + } else { + logging.Warn("irc.TOPIC(): topic change on unknown channel %s", + line.Args[0]) + } +} + +// Handle 311 whois reply +func (conn *Conn) h_311(line *Line) { + if !line.argslen(5) { + return + } + if nk := conn.st.GetNick(line.Args[1]); (nk != nil) && !conn.Me().Equals(nk) { + conn.st.NickInfo(line.Args[1], line.Args[2], line.Args[3], line.Args[5]) + } else { + logging.Warn("irc.311(): received WHOIS info for unknown nick %s", + line.Args[1]) + } +} + +// Handle 324 mode reply +func (conn *Conn) h_324(line *Line) { + if !line.argslen(2) { + return + } + if ch := conn.st.GetChannel(line.Args[1]); ch != nil { + conn.st.ChannelModes(line.Args[1], line.Args[2], line.Args[3:]...) + } else { + logging.Warn("irc.324(): received MODE settings for unknown channel %s", + line.Args[1]) + } +} + +// Handle 332 topic reply on join to channel +func (conn *Conn) h_332(line *Line) { + if !line.argslen(2) { + return + } + if ch := conn.st.GetChannel(line.Args[1]); ch != nil { + conn.st.Topic(line.Args[1], line.Args[2]) + } else { + logging.Warn("irc.332(): received TOPIC value for unknown channel %s", + line.Args[1]) + } +} + +// Handle 352 who reply +func (conn *Conn) h_352(line *Line) { + if !line.argslen(5) { + return + } + nk := conn.st.GetNick(line.Args[5]) + if nk == nil { + logging.Warn("irc.352(): received WHO reply for unknown nick %s", + line.Args[5]) + return + } + if conn.Me().Equals(nk) { + return + } + // XXX: do we care about the actual server the nick is on? + // or the hop count to this server? + // last arg contains " " + a := strings.SplitN(line.Args[len(line.Args)-1], " ", 2) + conn.st.NickInfo(nk.Nick, line.Args[2], line.Args[3], a[1]) + if !line.argslen(6) { + return + } + if idx := strings.Index(line.Args[6], "*"); idx != -1 { + conn.st.NickModes(nk.Nick, "+o") + } + if idx := strings.Index(line.Args[6], "B"); idx != -1 { + conn.st.NickModes(nk.Nick, "+B") + } + if idx := strings.Index(line.Args[6], "H"); idx != -1 { + conn.st.NickModes(nk.Nick, "+i") + } +} + +// Handle 353 names reply +func (conn *Conn) h_353(line *Line) { + if !line.argslen(2) { + return + } + if ch := conn.st.GetChannel(line.Args[2]); ch != nil { + nicks := strings.Split(line.Args[len(line.Args)-1], " ") + for _, nick := range nicks { + // UnrealIRCd's coders are lazy and leave a trailing space + if nick == "" { + continue + } + switch c := nick[0]; c { + case '~', '&', '@', '%', '+': + nick = nick[1:] + fallthrough + default: + if conn.st.GetNick(nick) == nil { + // we don't know this nick yet! + conn.st.NewNick(nick) + } + if _, ok := conn.st.IsOn(ch.Name, nick); !ok { + // This nick isn't associated with this channel yet! + conn.st.Associate(ch.Name, nick) + } + switch c { + case '~': + conn.st.ChannelModes(ch.Name, "+q", nick) + case '&': + conn.st.ChannelModes(ch.Name, "+a", nick) + case '@': + conn.st.ChannelModes(ch.Name, "+o", nick) + case '%': + conn.st.ChannelModes(ch.Name, "+h", nick) + case '+': + conn.st.ChannelModes(ch.Name, "+v", nick) + } + } + } + } else { + logging.Warn("irc.353(): received NAMES list for unknown channel %s", + line.Args[2]) + } +} + +// Handle 671 whois reply (nick connected via SSL) +func (conn *Conn) h_671(line *Line) { + if !line.argslen(1) { + return + } + if nk := conn.st.GetNick(line.Args[1]); nk != nil { + conn.st.NickModes(nk.Nick, "+z") + } else { + logging.Warn("irc.671(): received WHOIS SSL info for unknown nick %s", + line.Args[1]) + } +} diff --git a/vendor/github.com/fluffle/goirc/doc/rfc2811.txt b/vendor/github.com/fluffle/goirc/doc/rfc2811.txt new file mode 100644 index 0000000..7c2c4ea --- /dev/null +++ b/vendor/github.com/fluffle/goirc/doc/rfc2811.txt @@ -0,0 +1,1067 @@ + + + + + + +Network Working Group C. Kalt +Request for Comments: 2811 April 2000 +Updates: 1459 +Category: Informational + + + Internet Relay Chat: Channel Management + +Status of this Memo + + This memo provides information for the Internet community. It does + not specify an Internet standard of any kind. Distribution of this + memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2000). All Rights Reserved. + +Abstract + + One of the most notable characteristics of the IRC (Internet Relay + Chat) protocol is to allow for users to be grouped in forums, called + channels, providing a mean for multiple users to communicate + together. + + There was originally a unique type of channels, but with the years, + new types appeared either as a response to a need, or for + experimental purposes. + + This document specifies how channels, their characteristics and + properties are managed by IRC servers. + +Table of Contents + + 1. Introduction ............................................... 2 + 2. Channel Characteristics .................................... 3 + 2.1 Namespace .............................................. 3 + 2.2 Channel Scope .......................................... 3 + 2.3 Channel Properties ..................................... 4 + 2.4 Privileged Channel Members ............................. 4 + 2.4.1 Channel Operators ................................. 5 + 2.4.2 Channel Creator ................................... 5 + 3. Channel lifetime ........................................... 5 + 3.1 Standard channels ...................................... 5 + 3.2 Safe Channels .......................................... 6 + 4. Channel Modes .............................................. 7 + 4.1 Member Status .......................................... 7 + 4.1.1 "Channel Creator" Status .......................... 7 + + + +Kalt Informational [Page 1] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + + 4.1.2 Channel Operator Status ........................... 8 + 4.1.3 Voice Privilege ................................... 8 + 4.2 Channel Flags .......................................... 8 + 4.2.1 Anonymous Flag .................................... 8 + 4.2.2 Invite Only Flag .................................. 8 + 4.2.3 Moderated Channel Flag ............................ 9 + 4.2.4 No Messages To Channel From Clients On The Outside 9 + 4.2.5 Quiet Channel ..................................... 9 + 4.2.6 Private and Secret Channels ....................... 9 + 4.2.7 Server Reop Flag .................................. 10 + 4.2.8 Topic ............................................. 10 + 4.2.9 User Limit ........................................ 10 + 4.2.10 Channel Key ...................................... 10 + 4.3 Channel Access Control ................................. 10 + 4.3.1 Channel Ban and Exception ......................... 11 + 4.3.2 Channel Invitation ................................ 11 + 5. Current Implementations .................................... 11 + 5.1 Tracking Recently Used Channels ........................ 11 + 5.2 Safe Channels .......................................... 12 + 5.2.1 Channel Identifier ................................ 12 + 5.2.2 Channel Delay ..................................... 12 + 5.2.3 Abuse Window ...................................... 13 + 5.2.4 Preserving Sanity In The Name Space ............... 13 + 5.2.5 Server Reop Mechanism ............................. 13 + 6. Current problems ........................................... 14 + 6.1 Labels ................................................. 14 + 6.1.1 Channel Delay ..................................... 14 + 6.1.2 Safe Channels ..................................... 15 + 6.2 Mode Propagation Delays ................................ 15 + 6.3 Collisions And Channel Modes ........................... 15 + 6.4 Resource Exhaustion .................................... 16 + 7. Security Considerations .................................... 16 + 7.1 Access Control ......................................... 16 + 7.2 Channel Privacy ........................................ 16 + 7.3 Anonymity ............................................... 17 + 8. Current support and availability ........................... 17 + 9. Acknowledgements ........................................... 17 + 10. References ................................................ 18 + 11. Author's Address .......................................... 18 + 12. Full Copyright Statement ................................... 19 + +1. Introduction + + This document defines in detail on how channels are managed by the + IRC servers and will be mostly useful to people working on + implementing an IRC server. + + + + + +Kalt Informational [Page 2] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + + While the concepts defined here are an important part of IRC, they + remain non essential for implementing clients. While the trend seems + to be towards more and more complex and "intelligent" clients which + are able to take advantage of knowing the internal workings of + channels to provide the users with a more friendly interface, simple + clients can be implemented without reading this document. + + Many of the concepts defined here were designed with the IRC + architecture [IRC-ARCH] in mind and mostly make sense in this + context. However, many others could be applied to other + architectures in order to provide forums for a conferencing system. + + Finally, it is to be noted that IRC users may find some of the + following sections of interest, in particular sections 2 (Channel + Characteristics) and 4 (Channel Modes). + +2. Channel Characteristics + + A channel is a named group of one or more users which will all + receive messages addressed to that channel. A channel is + characterized by its name, properties and current members. + +2.1 Namespace + + Channels names are strings (beginning with a '&', '#', '+' or '!' + character) of length up to fifty (50) characters. Channel names are + case insensitive. + + Apart from the the requirement that the first character being either + '&', '#', '+' or '!' (hereafter called "channel prefix"). The only + restriction on a channel name is that it SHALL NOT contain any spaces + (' '), a control G (^G or ASCII 7), a comma (',' which is used as a + list item separator by the protocol). Also, a colon (':') is used as + a delimiter for the channel mask. The exact syntax of a channel name + is defined in "IRC Server Protocol" [IRC-SERVER]. + + The use of different prefixes effectively creates four (4) distinct + namespaces for channel names. This is important because of the + protocol limitations regarding namespaces (in general). See section + 6.1 (Labels) for more details on these limitations. + +2.2 Channel Scope + + A channel entity is known by one or more servers on the IRC network. + A user can only become member of a channel known by the server to + which the user is directly connected. The list of servers which know + + + + + +Kalt Informational [Page 3] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + + of the existence of a particular channel MUST be a contiguous part of + the IRC network, in order for the messages addressed to the channel + to be sent to all the channel members. + + Channels with '&' as prefix are local to the server where they are + created. + + Other channels are known to one (1) or more servers that are + connected to the network, depending on the channel mask: + + If there is no channel mask, then the channel is known to all + the servers. + + If there is a channel mask, then the channel MUST only be known + to servers which has a local user on the channel, and to its + neighbours if the mask matches both the local and neighbouring + server names. Since other servers have absolutely no knowledge of + the existence of such a channel, the area formed by the servers + having a name matching the mask has to be contiguous for the + channel to be known by all these servers. Channel masks are best + used in conjunction with server hostmasking [IRC-SERVER]. + +2.3 Channel Properties + + Each channel has its own properties, which are defined by channel + modes. Channel modes can be manipulated by the channel members. The + modes affect the way servers manage the channels. + + Channels with '+' as prefix do not support channel modes. This means + that all the modes are unset, with the exception of the 't' channel + flag which is set. + +2.4 Privileged Channel Members + + In order for the channel members to keep some control over a channel, + and some kind of sanity, some channel members are privileged. Only + these members are allowed to perform the following actions on the + channel: + + INVITE - Invite a client to an invite-only channel (mode +i) + KICK - Eject a client from the channel + MODE - Change the channel's mode, as well as + members' privileges + PRIVMSG - Sending messages to the channel (mode +n, +m, +v) + TOPIC - Change the channel topic in a mode +t channel + + + + + + +Kalt Informational [Page 4] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + +2.4.1 Channel Operators + + The channel operators (also referred to as a "chop" or "chanop") on a + given channel are considered to 'own' that channel. Ownership of a + channel is shared among channel operators. + + Channel operators are identified by the '@' symbol next to their + nickname whenever it is associated with a channel (i.e., replies to + the NAMES, WHO and WHOIS commands). + + Since channels starting with the character '+' as prefix do not + support channel modes, no member can therefore have the status of + channel operator. + +2.4.2 Channel Creator + + A user who creates a channel with the character '!' as prefix is + identified as the "channel creator". Upon creation of the channel, + this user is also given channel operator status. + + In recognition of this status, the channel creators are endowed with + the ability to toggle certain modes of the channel which channel + operators may not manipulate. + + A "channel creator" can be distinguished from a channel operator by + issuing the proper MODE command. See the "IRC Client Protocol" + [IRC-CLIENT] for more information on this topic. + +3. Channel lifetime + + In regard to the lifetime of a channel, there are typically two + groups of channels: standard channels which prefix is either '&', '#' + or '+', and "safe channels" which prefix is '!'. + +3.1 Standard channels + + These channels are created implicitly when the first user joins it, + and cease to exist when the last user leaves it. While the channel + exists, any client can reference the channel using the name of the + channel. + + The user creating a channel automatically becomes channel operator + with the notable exception of channels which name is prefixed by the + character '+', see section 4 (Channel modes). See section 2.4.1 + (Channel Operators) for more details on this title. + + + + + + +Kalt Informational [Page 5] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + + In order to avoid the creation of duplicate channels (typically when + the IRC network becomes disjoint because of a split between two + servers), channel names SHOULD NOT be allowed to be reused by a user + if a channel operator (See Section 2.4.1 (Channel Operators)) has + recently left the channel because of a network split. If this + happens, the channel name is temporarily unavailable. The duration + while a channel remains unavailable should be tuned on a per IRC + network basis. It is important to note that this prevents local + users from creating a channel using the same name, but does not + prevent the channel to be recreated by a remote user. The latter + typically happens when the IRC network rejoins. Obviously, this + mechanism only makes sense for channels which name begins with the + character '#', but MAY be used for channels which name begins with + the character '+'. This mechanism is commonly known as "Channel + Delay". + +3.2 Safe Channels + + Unlike other channels, "safe channels" are not implicitly created. A + user wishing to create such a channel MUST request the creation by + sending a special JOIN command to the server in which the channel + identifier (then unknown) is replaced by the character '!'. The + creation process for this type of channel is strictly controlled. + The user only chooses part of the channel name (known as the channel + "short name"), the server automatically prepends the user provided + name with a channel identifier consisting of five (5) characters. + The channel name resulting from the combination of these two elements + is unique, making the channel safe from abuses based on network + splits. + + The user who creates such a channel automatically becomes "channel + creator". See section 2.4.2 (Channel Creator) for more details on + this title. + + A server MUST NOT allow the creation of a new channel if another + channel with the same short name exists; or if another channel with + the same short name existed recently AND any of its member(s) left + because of a network split. Such channel ceases to exist after last + user leaves AND no other member recently left the channel because of + a network split. + + Unlike the mechanism described in section 5.2.2 (Channel Delay), in + this case, channel names do not become unavailable: these channels + may continue to exist after the last user left. Only the user + creating the channel becomes "channel creator", users joining an + existing empty channel do not automatically become "channel creator" + nor "channel operator". + + + + +Kalt Informational [Page 6] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + + To ensure the uniqueness of the channel names, the channel identifier + created by the server MUST follow specific rules. For more details + on this, see section 5.2.1 (Channel Identifier). + +4. Channel Modes + + The various modes available for channels are as follows: + + O - give "channel creator" status; + o - give/take channel operator privilege; + v - give/take the voice privilege; + + a - toggle the anonymous channel flag; + i - toggle the invite-only channel flag; + m - toggle the moderated channel; + n - toggle the no messages to channel from clients on the + outside; + q - toggle the quiet channel flag; + p - toggle the private channel flag; + s - toggle the secret channel flag; + r - toggle the server reop channel flag; + t - toggle the topic settable by channel operator only flag; + + k - set/remove the channel key (password); + l - set/remove the user limit to channel; + + b - set/remove ban mask to keep users out; + e - set/remove an exception mask to override a ban mask; + I - set/remove an invitation mask to automatically override + the invite-only flag; + + Unless mentioned otherwise below, all these modes can be manipulated + by "channel operators" by using the MODE command defined in "IRC + Client Protocol" [IRC-CLIENT]. + +4.1 Member Status + + The modes in this category take a channel member nickname as argument + and affect the privileges given to this user. + +4.1.1 "Channel Creator" Status + + The mode 'O' is only used in conjunction with "safe channels" and + SHALL NOT be manipulated by users. Servers use it to give the user + creating the channel the status of "channel creator". + + + + + + +Kalt Informational [Page 7] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + +4.1.2 Channel Operator Status + + The mode 'o' is used to toggle the operator status of a channel + member. + +4.1.3 Voice Privilege + + The mode 'v' is used to give and take voice privilege to/from a + channel member. Users with this privilege can talk on moderated + channels. (See section 4.2.3 (Moderated Channel Flag). + +4.2 Channel Flags + + The modes in this category are used to define properties which + affects how channels operate. + +4.2.1 Anonymous Flag + + The channel flag 'a' defines an anonymous channel. This means that + when a message sent to the channel is sent by the server to users, + and the origin is a user, then it MUST be masked. To mask the + message, the origin is changed to "anonymous!anonymous@anonymous." + (e.g., a user with the nickname "anonymous", the username "anonymous" + and from a host called "anonymous."). Because of this, servers MUST + forbid users from using the nickname "anonymous". Servers MUST also + NOT send QUIT messages for users leaving such channels to the other + channel members but generate a PART message instead. + + On channels with the character '&' as prefix, this flag MAY be + toggled by channel operators, but on channels with the character '!' + as prefix, this flag can be set (but SHALL NOT be unset) by the + "channel creator" only. This flag MUST NOT be made available on + other types of channels. + + Replies to the WHOIS, WHO and NAMES commands MUST NOT reveal the + presence of other users on channels for which the anonymous flag is + set. + +4.2.2 Invite Only Flag + + When the channel flag 'i' is set, new members are only accepted if + their mask matches Invite-list (See section 4.3.2) or they have been + invited by a channel operator. This flag also restricts the usage of + the INVITE command (See "IRC Client Protocol" [IRC-CLIENT]) to + channel operators. + + + + + + +Kalt Informational [Page 8] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + +4.2.3 Moderated Channel Flag + + The channel flag 'm' is used to control who may speak on a channel. + When it is set, only channel operators, and members who have been + given the voice privilege may send messages to the channel. + + This flag only affects users. + +4.2.4 No Messages To Channel From Clients On The Outside + + When the channel flag 'n' is set, only channel members MAY send + messages to the channel. + + This flag only affects users. + +4.2.5 Quiet Channel + + The channel flag 'q' is for use by servers only. When set, it + restricts the type of data sent to users about the channel + operations: other user joins, parts and nick changes are not sent. + From a user's point of view, the channel contains only one user. + + This is typically used to create special local channels on which the + server sends notices related to its operations. This was used as a + more efficient and flexible way to replace the user mode 's' defined + in RFC 1459 [IRC]. + +4.2.6 Private and Secret Channels + + The channel flag 'p' is used to mark a channel "private" and the + channel flag 's' to mark a channel "secret". Both properties are + similar and conceal the existence of the channel from other users. + + This means that there is no way of getting this channel's name from + the server without being a member. In other words, these channels + MUST be omitted from replies to queries like the WHOIS command. + + When a channel is "secret", in addition to the restriction above, the + server will act as if the channel does not exist for queries like the + TOPIC, LIST, NAMES commands. Note that there is one exception to + this rule: servers will correctly reply to the MODE command. + Finally, secret channels are not accounted for in the reply to the + LUSERS command (See "Internet Relay Chat: Client Protocol" [IRC- + CLIENT]) when the parameter is specified. + + + + + + + +Kalt Informational [Page 9] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + + The channel flags 'p' and 's' MUST NOT both be set at the same time. + If a MODE message originating from a server sets the flag 'p' and the + flag 's' is already set for the channel, the change is silently + ignored. This should only happen during a split healing phase + (mentioned in the "IRC Server Protocol" document [IRC-SERVER]). + +4.2.7 Server Reop Flag + + The channel flag 'r' is only available on channels which name begins + with the character '!' and MAY only be toggled by the "channel + creator". + + This flag is used to prevent a channel from having no channel + operator for an extended period of time. When this flag is set, any + channel that has lost all its channel operators for longer than the + "reop delay" period triggers a mechanism in servers to reop some or + all of the channel inhabitants. This mechanism is described more in + detail in section 5.2.4 (Channel Reop Mechanism). + +4.2.8 Topic + + The channel flag 't' is used to restrict the usage of the TOPIC + command to channel operators. + +4.2.9 User Limit + + A user limit may be set on channels by using the channel flag 'l'. + When the limit is reached, servers MUST forbid their local users to + join the channel. + + The value of the limit MUST only be made available to the channel + members in the reply sent by the server to a MODE query. + +4.2.10 Channel Key + + When a channel key is set (by using the mode 'k'), servers MUST + reject their local users request to join the channel unless this key + is given. + + The channel key MUST only be made visible to the channel members in + the reply sent by the server to a MODE query. + +4.3 Channel Access Control + + The last category of modes is used to control access to the channel, + they take a mask as argument. + + + + + +Kalt Informational [Page 10] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + + In order to reduce the size of the global database for control access + modes set for channels, servers MAY put a maximum limit on the number + of such modes set for a particular channel. If such restriction is + imposed, it MUST only affect user requests. The limit SHOULD be + homogeneous on a per IRC network basis. + +4.3.1 Channel Ban and Exception + + When a user requests to join a channel, his local server checks if + the user's address matches any of the ban masks set for the channel. + If a match is found, the user request is denied unless the address + also matches an exception mask set for the channel. + + Servers MUST NOT allow a channel member who is banned from the + channel to speak on the channel, unless this member is a channel + operator or has voice privilege. (See Section 4.1.3 (Voice + Privilege)). + + A user who is banned from a channel and who carries an invitation + sent by a channel operator is allowed to join the channel. + +4.3.2 Channel Invitation + + For channels which have the invite-only flag set (See Section 4.2.2 + (Invite Only Flag)), users whose address matches an invitation mask + set for the channel are allowed to join the channel without any + invitation. + +5. Current Implementations + + The only current implementation of these rules as part of the IRC + protocol is the IRC server, version 2.10. + + The rest of this section deals with issues that are mostly of + importance to those who wish to implement a server but some parts may + also be of interest for client writers. + +5.1 Tracking Recently Used Channels + + This mechanism is commonly known as "Channel Delay" and generally + only applies to channels which names is prefixed with the character + '#' (See Section 3.1 "Standard channels"). + + When a network split occurs, servers SHOULD keep track of which + channels lost a "channel operator" as the result of the break. These + channels are then in a special state which lasts for a certain period + of time. In this particular state, the channels cannot cease to + + + + +Kalt Informational [Page 11] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + + exist. If all the channel members leave the channel, the channel + becomes unavailable: the server local clients cannot join the channel + as long as it is empty. + + Once a channel is unavailable, it will become available again either + because a remote user has joined the channel (most likely because the + network is healing), or because the delay period has expired (in + which case the channel ceases to exist and may be re-created). + + The duration for which a channel death is delayed SHOULD be set + considering many factors among which are the size (user wise) of the + IRC network, and the usual duration of network splits. It SHOULD be + uniform on all servers for a given IRC network. + +5.2 Safe Channels + + This document introduces the notion of "safe channels". These + channels have a name prefixed with the character '!' and great effort + is made to avoid collisions in this name space. Collisions are not + impossible, however they are very unlikely. + +5.2.1 Channel Identifier + + The channel identifier is a function of the time. The current time + (as defined under UNIX by the number of seconds elapsed since + 00:00:00 GMT, January 1, 1970) is converted in a string of five (5) + characters using the following base: + "ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890" (each character has a decimal + value starting from 0 for 'A' to 35 for '0'). + + The channel identifier therefore has a periodicity of 36^5 seconds + (about 700 days). + +5.2.2 Channel Delay + + These channels MUST be subject to the "channel delay" mechanism + described in section 5.1 (Channel Delay). However, the mechanism is + slightly adapted to fit better. + + Servers MUST keep track of all such channels which lose members as + the result of a network split, no matter whether the user is a + "channel operator" or not. + + However, these channels do NOT ever become unavailable, it is always + possible to join them even when they are empty. + + + + + + +Kalt Informational [Page 12] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + +5.2.3 Abuse Window + + Because the periodicity is so long, attacks on a particular channel + (name) may only occur once in a very long while. However, with luck + and patience, it is still possible for a user to cause a channel + collision. In order to avoid this, servers MUST "look in the future" + and keep a list of channel names which identifier is about to be used + (in the coming few days for example). Such list should remain small, + not be a burden for servers to maintain and be used to avoid channel + collisions by preventing the re-creation of such channel for a longer + period of time than channel delay does. + + Eventually a server MAY choose to extend this procedure to forbid + creation of channels with the same shortname only (then ignoring the + channel identifier). + +5.2.4 Preserving Sanity In The Name Space + + The combination of the mechanisms described in sections 5.2.2 and + 5.2.3 makes it quite difficult for a user to create a channel + collision. However, another type of abuse consists of creating many + channels having the same shortname, but different identifiers. To + prevent this from happening, servers MUST forbid the creation of a + new channel which has the same shortname of a channel currently + existing. + +5.2.5 Server Reop Mechanism + + When a channel has been opless for longer than the "reop delay" + period and has the channel flag 'r' set (See Section 4.2.7 (Server + Reop Flag)), IRC servers are responsible for giving the channel + operator status randomly to some of the members. + + The exact logic used for this mechanism by the current implementation + is described below. Servers MAY use a different logic, but that it + is strongly RECOMMENDED that all servers use the same logic on a + particular IRC network to maintain coherence as well as fairness. + For the same reason, the "reop delay" SHOULD be uniform on all + servers for a given IRC network. As for the "channel delay", the + value of the "reop delay" SHOULD be set considering many factors + among which are the size (user wise) of the IRC network, and the + usual duration of network splits. + + a) the reop mechanism is triggered after a random time following the + expiration of the "reop delay". This should limit the eventuality + of the mechanism being triggered at the same time (for the same + channel) on two separate servers. + + + + +Kalt Informational [Page 13] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + + b) If the channel is small (five (5) users or less), and the "channel + delay" for this channel has expired, + Then reop all channel members if at least one member is local to + the server. + + c) If the channel is small (five (5) users or less), and the "channel + delay" for this channel has expired, and the "reop delay" has + expired for longer than its value, + Then reop all channel members. + + d) For other cases, reop at most one member on the channel, based on + some method build into the server. If you don't reop a member, the + method should be such that another server will probably op + someone. The method SHOULD be the same over the whole network. A + good heuristic could be just random reop. + (The current implementation actually tries to choose a member + local to the server who has not been idle for too long, eventually + postponing action, therefore letting other servers have a chance + to find a "not too idle" member. This is over complicated due to + the fact that servers only know the "idle" time of their local + users) + +6. Current problems + + There are a number of recognized problems with the way IRC channels + are managed. Some of these can be directly attributed to the rules + defined in this document, while others are the result of the + underlying "IRC Server Protocol" [IRC-SERVER]. Although derived from + RFC 1459 [IRC], this document introduces several novelties in an + attempt to solve some of the known problems. + +6.1 Labels + + This document defines one of the many labels used by the IRC + protocol. Although there are several distinct namespaces (based on + the channel name prefix), duplicates inside each of these are not + allowed. Currently, it is possible for users on different servers to + pick the label which may result in collisions (with the exception of + channels known to only one server where they can be averted). + +6.1.1 Channel Delay + + The channel delay mechanism described in section 5.1 (Tracking + Recently Used Channels) and used for channels prefixed with the + character '#' is a simple attempt at preventing collisions from + happening. Experience has shown that, under normal circumstances, it + + + + + +Kalt Informational [Page 14] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + + is very efficient; however, it obviously has severe limitations + keeping it from being an adequate solution to the problem discussed + here. + +6.1.2 Safe Channels + + "Safe channels" described in section 3.2 (Safe Channels) are a better + way to prevent collisions from happening as it prevents users from + having total control over the label they choose. The obvious + drawback for such labels is that they are not user friendly. + However, it is fairly trivial for a client program to improve on + this. + +6.2 Mode Propagation Delays + + Because of network delays induced by the network, and because each + server on the path is REQUIRED to check the validity of mode changes + (e.g., user exists and has the right privileges), it is not unusual + for a MODE message to only affect part of the network, often creating + a discrepancy between servers on the current state of a channel. + + While this may seem easy to fix (by having only the original server + check the validity of mode changes), it was decided not to do so for + various reasons. One concern is that servers cannot trust each + other, and that a misbehaving servers can easily be detected. This + way of doing so also stops wave effects on channels which are out of + synch when mode changes are issued from different directions. + +6.3 Collisions And Channel Modes + + The "Internet Relay Chat: Server Protocol" document [IRC-SERVER] + describes how channel data is exchanged when two servers connect to + each other. Channel collisions (either legitimate or not) are + treated as inclusive events, meaning that the resulting channel has + for members all the users who are members of the channel on either + server prior to the connection. + + Similarly, each server sends the channel modes to the other one. + Therefore, each server also receives these channel modes. There are + three types of modes for a given channel: flags, masks, and data. + The first two types are easy to deal with as they are either set or + unset. If such a mode is set on one server, it MUST be set on the + other server as a result of the connection. + + + + + + + + +Kalt Informational [Page 15] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + + As topics are not sent as part of this exchange, they are not a + problem. However, channel modes 'l' and 'k' are exchanged, and if + they are set on both servers prior to the connection, there is no + mechanism to decide which of the two values takes precedence. It is + left up to the users to fix the resulting discrepancy. + +6.4 Resource Exhaustion + + The mode based on masks defined in section 4.3 make the IRC servers + (and network) vulnerable to a simple abuse of the system: a single + channel operator can set as many different masks as possible on a + particular channel. This can easily cause the server to waste + memory, as well as network bandwidth (since the info is propagated to + other servers). For this reason it is RECOMMENDED that a limit be + put on the number of such masks per channels as mentioned in section + 4.3. + + Moreover, more complex mechanisms MAY be used to avoid having + redundant masks set for the same channel. + +7. Security Considerations + +7.1 Access Control + + One of the main ways to control access to a channel is to use masks + which are based on the username and hostname of the user connections. + This mechanism can only be efficient and safe if the IRC servers have + an accurate way of authenticating user connections, and if users + cannot easily get around it. While it is in theory possible to + implement such a strict authentication mechanism, most IRC networks + (especially public networks) do not have anything like this in place + and provide little guaranty about the accuracy of the username and + hostname for a particular client connection. + + Another way to control access is to use a channel key, but since this + key is sent in plaintext, it is vulnerable to traditional man in the + middle attacks. + +7.2 Channel Privacy + + Because channel collisions are treated as inclusive events (See + Section 6.3), it is possible for users to join a channel overriding + its access control settings. This method has long been used by + individuals to "take over" channels by "illegitimately" gaining + channel operator status on the channel. The same method can be used + to find out the exact list of members of a channel, as well as to + eventually receive some of the messages sent to the channel. + + + + +Kalt Informational [Page 16] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + +7.3 Anonymity + + The anonymous channel flag (See Section 4.2.1) can be used to render + all users on such channel "anonymous" by presenting all messages to + the channel as originating from a pseudo user which nickname is + "anonymous". This is done at the client-server level, and no + anonymity is provided at the server-server level. + + It should be obvious to readers, that the level of anonymity offered + is quite poor and insecure, and that clients SHOULD display strong + warnings for users joining such channels. + +8. Current support and availability + + Mailing lists for IRC related discussion: + General discussion: ircd-users@irc.org + Protocol development: ircd-dev@irc.org + + Software implementations: + ftp://ftp.irc.org/irc/server + ftp://ftp.funet.fi/pub/unix/irc + ftp://coombs.anu.edu.au/pub/irc + + Newsgroup: alt.irc + +9. Acknowledgements + + Parts of this document were copied from the RFC 1459 [IRC] which + first formally documented the IRC Protocol. It has also benefited + from many rounds of review and comments. In particular, the + following people have made significant contributions to this + document: + + Matthew Green, Michael Neumayer, Volker Paulsen, Kurt Roeckx, Vesa + Ruokonen, Magnus Tjernstrom, Stefan Zehl. + + + + + + + + + + + + + + + + +Kalt Informational [Page 17] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + +10. References + + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [IRC] Oikarinen, J. and D. Reed, "Internet Relay Chat + Protocol", RFC 1459, May 1993. + + [IRC-ARCH] Kalt, C., "Internet Relay Chat: Architecture", RFC 2810, + April 2000. + + [IRC-CLIENT] Kalt, C., "Internet Relay Chat: Client Protocol", RFC + 2812, April 2000. + + [IRC-SERVER] Kalt, C., "Internet Relay Chat: Server Protocol", RFC + 2813, April 2000. + +11. Author's Address + + Christophe Kalt + 99 Teaneck Rd, Apt #117 + Ridgefield Park, NJ 07660 + USA + + EMail: kalt@stealth.net + + + + + + + + + + + + + + + + + + + + + + + + + + +Kalt Informational [Page 18] + +RFC 2811 Internet Relay Chat: Channel Management April 2000 + + +12. Full Copyright Statement + + Copyright (C) The Internet Society (2000). All Rights Reserved. + + This document and translations of it may be copied and furnished to + others, and derivative works that comment on or otherwise explain it + or assist in its implementation may be prepared, copied, published + and distributed, in whole or in part, without restriction of any + kind, provided that the above copyright notice and this paragraph are + included on all such copies and derivative works. However, this + document itself may not be modified in any way, such as by removing + the copyright notice or references to the Internet Society or other + Internet organizations, except as needed for the purpose of + developing Internet standards in which case the procedures for + copyrights defined in the Internet Standards process must be + followed, or as required to translate it into languages other than + English. + + The limited permissions granted above are perpetual and will not be + revoked by the Internet Society or its successors or assigns. + + This document and the information contained herein is provided on an + "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING + TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION + HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Acknowledgement + + Funding for the RFC Editor function is currently provided by the + Internet Society. + + + + + + + + + + + + + + + + + + + +Kalt Informational [Page 19] + diff --git a/vendor/github.com/fluffle/goirc/doc/rfc2812.txt b/vendor/github.com/fluffle/goirc/doc/rfc2812.txt new file mode 100644 index 0000000..7f1b162 --- /dev/null +++ b/vendor/github.com/fluffle/goirc/doc/rfc2812.txt @@ -0,0 +1,3531 @@ + + + + + + +Network Working Group C. Kalt +Request for Comments: 2812 April 2000 +Updates: 1459 +Category: Informational + + + Internet Relay Chat: Client Protocol + +Status of this Memo + + This memo provides information for the Internet community. It does + not specify an Internet standard of any kind. Distribution of this + memo is unlimited. + +Copyright Notice + + Copyright (C) The Internet Society (2000). All Rights Reserved. + +IESG NOTE: + + The IRC protocol itself enables several possibilities of transferring + data between clients, and just like with other transfer mechanisms + like email, the receiver of the data has to be careful about how the + data is handled. For more information on security issues with the IRC + protocol, see for example http://www.irchelp.org/irchelp/security/. + +Abstract + + The IRC (Internet Relay Chat) protocol is for use with text based + conferencing; the simplest client being any socket program capable of + connecting to the server. + + This document defines the Client Protocol, and assumes that the + reader is familiar with the IRC Architecture [IRC-ARCH]. + +Table of Contents + + 1. Labels ..................................................... 3 + 1.1 Servers ................................................ 3 + 1.2 Clients ................................................ 3 + 1.2.1 Users ............................................. 4 + 1.2.1.1 Operators .................................... 4 + 1.2.2 Services .......................................... 4 + 1.3 Channels ............................................... 4 + 2. The IRC Client Specification ............................... 5 + 2.1 Overview ............................................... 5 + 2.2 Character codes ........................................ 5 + 2.3 Messages ............................................... 5 + + + +Kalt Informational [Page 1] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + 2.3.1 Message format in Augmented BNF ................... 6 + 2.4 Numeric replies ........................................ 8 + 2.5 Wildcard expressions ................................... 9 + 3. Message Details ............................................ 9 + 3.1 Connection Registration ................................ 10 + 3.1.1 Password message .................................. 10 + 3.1.2 Nick message ...................................... 10 + 3.1.3 User message ...................................... 11 + 3.1.4 Oper message ...................................... 12 + 3.1.5 User mode message ................................. 12 + 3.1.6 Service message ................................... 13 + 3.1.7 Quit .............................................. 14 + 3.1.8 Squit ............................................. 15 + 3.2 Channel operations ..................................... 15 + 3.2.1 Join message ...................................... 16 + 3.2.2 Part message ...................................... 17 + 3.2.3 Channel mode message .............................. 18 + 3.2.4 Topic message ..................................... 19 + 3.2.5 Names message ..................................... 20 + 3.2.6 List message ...................................... 21 + 3.2.7 Invite message .................................... 21 + 3.2.8 Kick command ...................................... 22 + 3.3 Sending messages ....................................... 23 + 3.3.1 Private messages .................................. 23 + 3.3.2 Notice ............................................ 24 + 3.4 Server queries and commands ............................ 25 + 3.4.1 Motd message ...................................... 25 + 3.4.2 Lusers message .................................... 25 + 3.4.3 Version message ................................... 26 + 3.4.4 Stats message ..................................... 26 + 3.4.5 Links message ..................................... 27 + 3.4.6 Time message ...................................... 28 + 3.4.7 Connect message ................................... 28 + 3.4.8 Trace message ..................................... 29 + 3.4.9 Admin command ..................................... 30 + 3.4.10 Info command ...................................... 31 + 3.5 Service Query and Commands ............................. 31 + 3.5.1 Servlist message .................................. 31 + 3.5.2 Squery ............................................ 32 + 3.6 User based queries ..................................... 32 + 3.6.1 Who query ......................................... 32 + 3.6.2 Whois query ....................................... 33 + 3.6.3 Whowas ............................................ 34 + 3.7 Miscellaneous messages ................................. 34 + 3.7.1 Kill message ...................................... 35 + 3.7.2 Ping message ...................................... 36 + 3.7.3 Pong message ...................................... 37 + 3.7.4 Error ............................................. 37 + + + +Kalt Informational [Page 2] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + 4. Optional features .......................................... 38 + 4.1 Away ................................................... 38 + 4.2 Rehash message ......................................... 39 + 4.3 Die message ............................................ 39 + 4.4 Restart message ........................................ 40 + 4.5 Summon message ......................................... 40 + 4.6 Users .................................................. 41 + 4.7 Operwall message ....................................... 41 + 4.8 Userhost message ....................................... 42 + 4.9 Ison message ........................................... 42 + 5. Replies .................................................... 43 + 5.1 Command responses ...................................... 43 + 5.2 Error Replies .......................................... 53 + 5.3 Reserved numerics ...................................... 59 + 6. Current implementations .................................... 60 + 7. Current problems ........................................... 60 + 7.1 Nicknames .............................................. 60 + 7.2 Limitation of wildcards ................................ 61 + 7.3 Security considerations ................................ 61 + 8. Current support and availability ........................... 61 + 9. Acknowledgements ........................................... 61 + 10. References ................................................ 62 + 11. Author's Address .......................................... 62 + 12. Full Copyright Statement .................................. 63 + +1. Labels + + This section defines the identifiers used for the various components + of the IRC protocol. + +1.1 Servers + + Servers are uniquely identified by their name, which has a maximum + length of sixty three (63) characters. See the protocol grammar + rules (section 2.3.1) for what may and may not be used in a server + name. + +1.2 Clients + + For each client all servers MUST have the following information: a + netwide unique identifier (whose format depends on the type of + client) and the server which introduced the client. + + + + + + + + + +Kalt Informational [Page 3] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + +1.2.1 Users + + Each user is distinguished from other users by a unique nickname + having a maximum length of nine (9) characters. See the protocol + grammar rules (section 2.3.1) for what may and may not be used in a + nickname. + + While the maximum length is limited to nine characters, clients + SHOULD accept longer strings as they may become used in future + evolutions of the protocol. + +1.2.1.1 Operators + + To allow a reasonable amount of order to be kept within the IRC + network, a special class of users (operators) is allowed to perform + general maintenance functions on the network. Although the powers + granted to an operator can be considered as 'dangerous', they are + nonetheless often necessary. Operators SHOULD be able to perform + basic network tasks such as disconnecting and reconnecting servers as + needed. In recognition of this need, the protocol discussed herein + provides for operators only to be able to perform such functions. + See sections 3.1.8 (SQUIT) and 3.4.7 (CONNECT). + + A more controversial power of operators is the ability to remove a + user from the connected network by 'force', i.e., operators are able + to close the connection between any client and server. The + justification for this is very delicate since its abuse is both + destructive and annoying, and its benefits close to inexistent. For + further details on this type of action, see section 3.7.1 (KILL). + +1.2.2 Services + + Each service is distinguished from other services by a service name + composed of a nickname and a server name. As for users, the nickname + has a maximum length of nine (9) characters. See the protocol + grammar rules (section 2.3.1) for what may and may not be used in a + nickname. + +1.3 Channels + + Channels names are strings (beginning with a '&', '#', '+' or '!' + character) of length up to fifty (50) characters. Apart from the + requirement that the first character is either '&', '#', '+' or '!', + the only restriction on a channel name is that it SHALL NOT contain + any spaces (' '), a control G (^G or ASCII 7), a comma (','). Space + is used as parameter separator and command is used as a list item + separator by the protocol). A colon (':') can also be used as a + delimiter for the channel mask. Channel names are case insensitive. + + + +Kalt Informational [Page 4] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + See the protocol grammar rules (section 2.3.1) for the exact syntax + of a channel name. + + Each prefix characterizes a different channel type. The definition + of the channel types is not relevant to the client-server protocol + and thus it is beyond the scope of this document. More details can + be found in "Internet Relay Chat: Channel Management" [IRC-CHAN]. + +2. The IRC Client Specification + +2.1 Overview + + The protocol as described herein is for use only with client to + server connections when the client registers as a user. + +2.2 Character codes + + No specific character set is specified. The protocol is based on a + set of codes which are composed of eight (8) bits, making up an + octet. Each message may be composed of any number of these octets; + however, some octet values are used for control codes, which act as + message delimiters. + + Regardless of being an 8-bit protocol, the delimiters and keywords + are such that protocol is mostly usable from US-ASCII terminal and a + telnet connection. + + Because of IRC's Scandinavian origin, the characters {}|^ are + considered to be the lower case equivalents of the characters []\~, + respectively. This is a critical issue when determining the + equivalence of two nicknames or channel names. + +2.3 Messages + + Servers and clients send each other messages, which may or may not + generate a reply. If the message contains a valid command, as + described in later sections, the client should expect a reply as + specified but it is not advised to wait forever for the reply; client + to server and server to server communication is essentially + asynchronous by nature. + + Each IRC message may consist of up to three main parts: the prefix + (OPTIONAL), the command, and the command parameters (maximum of + fifteen (15)). The prefix, command, and all parameters are separated + by one ASCII space character (0x20) each. + + + + + + +Kalt Informational [Page 5] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + The presence of a prefix is indicated with a single leading ASCII + colon character (':', 0x3b), which MUST be the first character of the + message itself. There MUST be NO gap (whitespace) between the colon + and the prefix. The prefix is used by servers to indicate the true + origin of the message. If the prefix is missing from the message, it + is assumed to have originated from the connection from which it was + received from. Clients SHOULD NOT use a prefix when sending a + message; if they use one, the only valid prefix is the registered + nickname associated with the client. + + The command MUST either be a valid IRC command or a three (3) digit + number represented in ASCII text. + + IRC messages are always lines of characters terminated with a CR-LF + (Carriage Return - Line Feed) pair, and these messages SHALL NOT + exceed 512 characters in length, counting all characters including + the trailing CR-LF. Thus, there are 510 characters maximum allowed + for the command and its parameters. There is no provision for + continuation of message lines. See section 6 for more details about + current implementations. + +2.3.1 Message format in Augmented BNF + + The protocol messages must be extracted from the contiguous stream of + octets. The current solution is to designate two characters, CR and + LF, as message separators. Empty messages are silently ignored, + which permits use of the sequence CR-LF between messages without + extra problems. + + The extracted message is parsed into the components , + and list of parameters (). + + The Augmented BNF representation for this is: + + message = [ ":" prefix SPACE ] command [ params ] crlf + prefix = servername / ( nickname [ [ "!" user ] "@" host ] ) + command = 1*letter / 3digit + params = *14( SPACE middle ) [ SPACE ":" trailing ] + =/ 14( SPACE middle ) [ SPACE [ ":" ] trailing ] + + nospcrlfcl = %x01-09 / %x0B-0C / %x0E-1F / %x21-39 / %x3B-FF + ; any octet except NUL, CR, LF, " " and ":" + middle = nospcrlfcl *( ":" / nospcrlfcl ) + trailing = *( ":" / " " / nospcrlfcl ) + + SPACE = %x20 ; space character + crlf = %x0D %x0A ; "carriage return" "linefeed" + + + + +Kalt Informational [Page 6] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + NOTES: + 1) After extracting the parameter list, all parameters are equal + whether matched by or . is just a + syntactic trick to allow SPACE within the parameter. + + 2) The NUL (%x00) character is not special in message framing, and + basically could end up inside a parameter, but it would cause + extra complexities in normal C string handling. Therefore, NUL + is not allowed within messages. + + Most protocol messages specify additional semantics and syntax for + the extracted parameter strings dictated by their position in the + list. For example, many server commands will assume that the first + parameter after the command is the list of targets, which can be + described with: + + target = nickname / server + msgtarget = msgto *( "," msgto ) + msgto = channel / ( user [ "%" host ] "@" servername ) + msgto =/ ( user "%" host ) / targetmask + msgto =/ nickname / ( nickname "!" user "@" host ) + channel = ( "#" / "+" / ( "!" channelid ) / "&" ) chanstring + [ ":" chanstring ] + servername = hostname + host = hostname / hostaddr + hostname = shortname *( "." shortname ) + shortname = ( letter / digit ) *( letter / digit / "-" ) + *( letter / digit ) + ; as specified in RFC 1123 [HNAME] + hostaddr = ip4addr / ip6addr + ip4addr = 1*3digit "." 1*3digit "." 1*3digit "." 1*3digit + ip6addr = 1*hexdigit 7( ":" 1*hexdigit ) + ip6addr =/ "0:0:0:0:0:" ( "0" / "FFFF" ) ":" ip4addr + nickname = ( letter / special ) *8( letter / digit / special / "-" ) + targetmask = ( "$" / "#" ) mask + ; see details on allowed masks in section 3.3.1 + chanstring = %x01-07 / %x08-09 / %x0B-0C / %x0E-1F / %x21-2B + chanstring =/ %x2D-39 / %x3B-FF + ; any octet except NUL, BELL, CR, LF, " ", "," and ":" + channelid = 5( %x41-5A / digit ) ; 5( A-Z / 0-9 ) + + + + + + + + + + + +Kalt Informational [Page 7] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + Other parameter syntaxes are: + + user = 1*( %x01-09 / %x0B-0C / %x0E-1F / %x21-3F / %x41-FF ) + ; any octet except NUL, CR, LF, " " and "@" + key = 1*23( %x01-05 / %x07-08 / %x0C / %x0E-1F / %x21-7F ) + ; any 7-bit US_ASCII character, + ; except NUL, CR, LF, FF, h/v TABs, and " " + letter = %x41-5A / %x61-7A ; A-Z / a-z + digit = %x30-39 ; 0-9 + hexdigit = digit / "A" / "B" / "C" / "D" / "E" / "F" + special = %x5B-60 / %x7B-7D + ; "[", "]", "\", "`", "_", "^", "{", "|", "}" + + NOTES: + 1) The syntax is given here for the sole purpose of + indicating the format to follow for IP addresses. This + reflects the fact that the only available implementations of + this protocol uses TCP/IP as underlying network protocol but is + not meant to prevent other protocols to be used. + + 2) has a maximum length of 63 characters. This is a + limitation of the protocol as internet hostnames (in + particular) can be longer. Such restriction is necessary + because IRC messages are limited to 512 characters in length. + Clients connecting from a host which name is longer than 63 + characters are registered using the host (numeric) address + instead of the host name. + + 3) Some parameters used in the following sections of this + documents are not defined here as there is nothing specific + about them besides the name that is used for convenience. + These parameters follow the general syntax defined for + . + +2.4 Numeric replies + + Most of the messages sent to the server generate a reply of some + sort. The most common reply is the numeric reply, used for both + errors and normal replies. The numeric reply MUST be sent as one + message consisting of the sender prefix, the three-digit numeric, and + the target of the reply. A numeric reply is not allowed to originate + from a client. In all other respects, a numeric reply is just like a + normal message, except that the keyword is made up of 3 numeric + digits rather than a string of letters. A list of different replies + is supplied in section 5 (Replies). + + + + + + +Kalt Informational [Page 8] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + +2.5 Wildcard expressions + + When wildcards are allowed in a string, it is referred as a "mask". + + For string matching purposes, the protocol allows the use of two + special characters: '?' (%x3F) to match one and only one character, + and '*' (%x2A) to match any number of any characters. These two + characters can be escaped using the character '\' (%x5C). + + The Augmented BNF syntax for this is: + + mask = *( nowild / noesc wildone / noesc wildmany ) + wildone = %x3F + wildmany = %x2A + nowild = %x01-29 / %x2B-3E / %x40-FF + ; any octet except NUL, "*", "?" + noesc = %x01-5B / %x5D-FF + ; any octet except NUL and "\" + matchone = %x01-FF + ; matches wildone + matchmany = *matchone + ; matches wildmany + + Examples: + + a?c ; Matches any string of 3 characters in length starting + with "a" and ending with "c" + + a*c ; Matches any string of at least 2 characters in length + starting with "a" and ending with "c" + +3. Message Details + + On the following pages there are descriptions of each message + recognized by the IRC server and client. All commands described in + this section MUST be implemented by any server for this protocol. + + Where the reply ERR_NOSUCHSERVER is returned, it means that the + target of the message could not be found. The server MUST NOT send + any other replies after this error for that command. + + The server to which a client is connected is required to parse the + complete message, and return any appropriate errors. + + If multiple parameters is presented, then each MUST be checked for + validity and appropriate responses MUST be sent back to the client. + In the case of incorrect messages which use parameter lists with + comma as an item separator, a reply MUST be sent for each item. + + + +Kalt Informational [Page 9] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + +3.1 Connection Registration + + The commands described here are used to register a connection with an + IRC server as a user as well as to correctly disconnect. + + A "PASS" command is not required for a client connection to be + registered, but it MUST precede the latter of the NICK/USER + combination (for a user connection) or the SERVICE command (for a + service connection). The RECOMMENDED order for a client to register + is as follows: + + 1. Pass message + 2. Nick message 2. Service message + 3. User message + + Upon success, the client will receive an RPL_WELCOME (for users) or + RPL_YOURESERVICE (for services) message indicating that the + connection is now registered and known the to the entire IRC network. + The reply message MUST contain the full client identifier upon which + it was registered. + +3.1.1 Password message + + Command: PASS + Parameters: + + The PASS command is used to set a 'connection password'. The + optional password can and MUST be set before any attempt to register + the connection is made. Currently this requires that user send a + PASS command before sending the NICK/USER combination. + + Numeric Replies: + + ERR_NEEDMOREPARAMS ERR_ALREADYREGISTRED + + Example: + + PASS secretpasswordhere + +3.1.2 Nick message + + + Command: NICK + Parameters: + + NICK command is used to give user a nickname or change the existing + one. + + + + +Kalt Informational [Page 10] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + Numeric Replies: + + ERR_NONICKNAMEGIVEN ERR_ERRONEUSNICKNAME + ERR_NICKNAMEINUSE ERR_NICKCOLLISION + ERR_UNAVAILRESOURCE ERR_RESTRICTED + + Examples: + + NICK Wiz ; Introducing new nick "Wiz" if session is + still unregistered, or user changing his + nickname to "Wiz" + + :WiZ!jto@tolsun.oulu.fi NICK Kilroy + ; Server telling that WiZ changed his + nickname to Kilroy. + +3.1.3 User message + + Command: USER + Parameters: + + The USER command is used at the beginning of connection to specify + the username, hostname and realname of a new user. + + The parameter should be a numeric, and can be used to + automatically set user modes when registering with the server. This + parameter is a bitmask, with only 2 bits having any signification: if + the bit 2 is set, the user mode 'w' will be set and if the bit 3 is + set, the user mode 'i' will be set. (See Section 3.1.5 "User + Modes"). + + The may contain space characters. + + Numeric Replies: + + ERR_NEEDMOREPARAMS ERR_ALREADYREGISTRED + + Example: + + USER guest 0 * :Ronnie Reagan ; User registering themselves with a + username of "guest" and real name + "Ronnie Reagan". + + USER guest 8 * :Ronnie Reagan ; User registering themselves with a + username of "guest" and real name + "Ronnie Reagan", and asking to be set + invisible. + + + + +Kalt Informational [Page 11] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + +3.1.4 Oper message + + Command: OPER + Parameters: + + A normal user uses the OPER command to obtain operator privileges. + The combination of and are REQUIRED to gain + Operator privileges. Upon success, the user will receive a MODE + message (see section 3.1.5) indicating the new user modes. + + Numeric Replies: + + ERR_NEEDMOREPARAMS RPL_YOUREOPER + ERR_NOOPERHOST ERR_PASSWDMISMATCH + + Example: + + OPER foo bar ; Attempt to register as an operator + using a username of "foo" and "bar" + as the password. + +3.1.5 User mode message + + Command: MODE + Parameters: + *( ( "+" / "-" ) *( "i" / "w" / "o" / "O" / "r" ) ) + + The user MODE's are typically changes which affect either how the + client is seen by others or what 'extra' messages the client is sent. + + A user MODE command MUST only be accepted if both the sender of the + message and the nickname given as a parameter are both the same. If + no other parameter is given, then the server will return the current + settings for the nick. + + The available modes are as follows: + + a - user is flagged as away; + i - marks a users as invisible; + w - user receives wallops; + r - restricted user connection; + o - operator flag; + O - local operator flag; + s - marks a user for receipt of server notices. + + Additional modes may be available later on. + + + + + +Kalt Informational [Page 12] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + The flag 'a' SHALL NOT be toggled by the user using the MODE command, + instead use of the AWAY command is REQUIRED. + + If a user attempts to make themselves an operator using the "+o" or + "+O" flag, the attempt SHOULD be ignored as users could bypass the + authentication mechanisms of the OPER command. There is no + restriction, however, on anyone `deopping' themselves (using "-o" or + "-O"). + + On the other hand, if a user attempts to make themselves unrestricted + using the "-r" flag, the attempt SHOULD be ignored. There is no + restriction, however, on anyone `deopping' themselves (using "+r"). + This flag is typically set by the server upon connection for + administrative reasons. While the restrictions imposed are left up + to the implementation, it is typical that a restricted user not be + allowed to change nicknames, nor make use of the channel operator + status on channels. + + The flag 's' is obsolete but MAY still be used. + + Numeric Replies: + + ERR_NEEDMOREPARAMS ERR_USERSDONTMATCH + ERR_UMODEUNKNOWNFLAG RPL_UMODEIS + + Examples: + + MODE WiZ -w ; Command by WiZ to turn off + reception of WALLOPS messages. + + MODE Angel +i ; Command from Angel to make herself + invisible. + + MODE WiZ -o ; WiZ 'deopping' (removing operator + status). + +3.1.6 Service message + + Command: SERVICE + Parameters: + + + The SERVICE command to register a new service. Command parameters + specify the service nickname, distribution, type and info of a new + service. + + + + + + +Kalt Informational [Page 13] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + The parameter is used to specify the visibility of a + service. The service may only be known to servers which have a name + matching the distribution. For a matching server to have knowledge + of the service, the network path between that server and the server + on which the service is connected MUST be composed of servers which + names all match the mask. + + The parameter is currently reserved for future usage. + + Numeric Replies: + + ERR_ALREADYREGISTRED ERR_NEEDMOREPARAMS + ERR_ERRONEUSNICKNAME + RPL_YOURESERVICE RPL_YOURHOST + RPL_MYINFO + + Example: + + SERVICE dict * *.fr 0 0 :French Dictionary ; Service registering + itself with a name of "dict". This + service will only be available on + servers which name matches "*.fr". + +3.1.7 Quit + + Command: QUIT + Parameters: [ ] + + A client session is terminated with a quit message. The server + acknowledges this by sending an ERROR message to the client. + + Numeric Replies: + + None. + + Example: + + QUIT :Gone to have lunch ; Preferred message format. + + :syrk!kalt@millennium.stealth.net QUIT :Gone to have lunch ; User + syrk has quit IRC to have lunch. + + + + + + + + + + +Kalt Informational [Page 14] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + +3.1.8 Squit + + Command: SQUIT + Parameters: + + The SQUIT command is available only to operators. It is used to + disconnect server links. Also servers can generate SQUIT messages on + error conditions. A SQUIT message may also target a remote server + connection. In this case, the SQUIT message will simply be sent to + the remote server without affecting the servers in between the + operator and the remote server. + + The SHOULD be supplied by all operators who execute a SQUIT + for a remote server. The server ordered to disconnect its peer + generates a WALLOPS message with included, so that other + users may be aware of the reason of this action. + + Numeric replies: + + ERR_NOPRIVILEGES ERR_NOSUCHSERVER + ERR_NEEDMOREPARAMS + + Examples: + + SQUIT tolsun.oulu.fi :Bad Link ? ; Command to uplink of the server + tolson.oulu.fi to terminate its + connection with comment "Bad Link". + + :Trillian SQUIT cm22.eng.umd.edu :Server out of control ; Command + from Trillian from to disconnect + "cm22.eng.umd.edu" from the net with + comment "Server out of control". + +3.2 Channel operations + + This group of messages is concerned with manipulating channels, their + properties (channel modes), and their contents (typically users). + For this reason, these messages SHALL NOT be made available to + services. + + All of these messages are requests which will or will not be granted + by the server. The server MUST send a reply informing the user + whether the request was granted, denied or generated an error. When + the server grants the request, the message is typically sent back + (eventually reformatted) to the user with the prefix set to the user + itself. + + + + + +Kalt Informational [Page 15] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + The rules governing how channels are managed are enforced by the + servers. These rules are beyond the scope of this document. More + details are found in "Internet Relay Chat: Channel Management" [IRC- + CHAN]. + +3.2.1 Join message + + Command: JOIN + Parameters: ( *( "," ) [ *( "," ) ] ) + / "0" + + The JOIN command is used by a user to request to start listening to + the specific channel. Servers MUST be able to parse arguments in the + form of a list of target, but SHOULD NOT use lists when sending JOIN + messages to clients. + + Once a user has joined a channel, he receives information about + all commands his server receives affecting the channel. This + includes JOIN, MODE, KICK, PART, QUIT and of course PRIVMSG/NOTICE. + This allows channel members to keep track of the other channel + members, as well as channel modes. + + If a JOIN is successful, the user receives a JOIN message as + confirmation and is then sent the channel's topic (using RPL_TOPIC) and + the list of users who are on the channel (using RPL_NAMREPLY), which + MUST include the user joining. + + Note that this message accepts a special argument ("0"), which is + a special request to leave all channels the user is currently a member + of. The server will process this message as if the user had sent + a PART command (See Section 3.2.2) for each channel he is a member + of. + + Numeric Replies: + + ERR_NEEDMOREPARAMS ERR_BANNEDFROMCHAN + ERR_INVITEONLYCHAN ERR_BADCHANNELKEY + ERR_CHANNELISFULL ERR_BADCHANMASK + ERR_NOSUCHCHANNEL ERR_TOOMANYCHANNELS + ERR_TOOMANYTARGETS ERR_UNAVAILRESOURCE + RPL_TOPIC + + Examples: + + JOIN #foobar ; Command to join channel #foobar. + + JOIN &foo fubar ; Command to join channel &foo using + key "fubar". + + + +Kalt Informational [Page 16] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + JOIN #foo,&bar fubar ; Command to join channel #foo using + key "fubar" and &bar using no key. + + JOIN #foo,#bar fubar,foobar ; Command to join channel #foo using + key "fubar", and channel #bar using + key "foobar". + + JOIN #foo,#bar ; Command to join channels #foo and + #bar. + + JOIN 0 ; Leave all currently joined + channels. + + :WiZ!jto@tolsun.oulu.fi JOIN #Twilight_zone ; JOIN message from WiZ + on channel #Twilight_zone + +3.2.2 Part message + + Command: PART + Parameters: *( "," ) [ ] + + The PART command causes the user sending the message to be removed + from the list of active members for all given channels listed in the + parameter string. If a "Part Message" is given, this will be sent + instead of the default message, the nickname. This request is always + granted by the server. + + Servers MUST be able to parse arguments in the form of a list of + target, but SHOULD NOT use lists when sending PART messages to + clients. + + Numeric Replies: + + ERR_NEEDMOREPARAMS ERR_NOSUCHCHANNEL + ERR_NOTONCHANNEL + + Examples: + + PART #twilight_zone ; Command to leave channel + "#twilight_zone" + + PART #oz-ops,&group5 ; Command to leave both channels + "&group5" and "#oz-ops". + + :WiZ!jto@tolsun.oulu.fi PART #playzone :I lost + ; User WiZ leaving channel + "#playzone" with the message "I + lost". + + + +Kalt Informational [Page 17] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + +3.2.3 Channel mode message + + Command: MODE + Parameters: *( ( "-" / "+" ) * * ) + + The MODE command is provided so that users may query and change the + characteristics of a channel. For more details on available modes + and their uses, see "Internet Relay Chat: Channel Management" [IRC- + CHAN]. Note that there is a maximum limit of three (3) changes per + command for modes that take a parameter. + + Numeric Replies: + + ERR_NEEDMOREPARAMS ERR_KEYSET + ERR_NOCHANMODES ERR_CHANOPRIVSNEEDED + ERR_USERNOTINCHANNEL ERR_UNKNOWNMODE + RPL_CHANNELMODEIS + RPL_BANLIST RPL_ENDOFBANLIST + RPL_EXCEPTLIST RPL_ENDOFEXCEPTLIST + RPL_INVITELIST RPL_ENDOFINVITELIST + RPL_UNIQOPIS + + The following examples are given to help understanding the syntax of + the MODE command, but refer to modes defined in "Internet Relay Chat: + Channel Management" [IRC-CHAN]. + + Examples: + + MODE #Finnish +imI *!*@*.fi ; Command to make #Finnish channel + moderated and 'invite-only' with user + with a hostname matching *.fi + automatically invited. + + MODE #Finnish +o Kilroy ; Command to give 'chanop' privileges + to Kilroy on channel #Finnish. + + MODE #Finnish +v Wiz ; Command to allow WiZ to speak on + #Finnish. + + MODE #Fins -s ; Command to remove 'secret' flag + from channel #Fins. + + MODE #42 +k oulu ; Command to set the channel key to + "oulu". + + MODE #42 -k oulu ; Command to remove the "oulu" + channel key on channel "#42". + + + + +Kalt Informational [Page 18] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + MODE #eu-opers +l 10 ; Command to set the limit for the + number of users on channel + "#eu-opers" to 10. + + :WiZ!jto@tolsun.oulu.fi MODE #eu-opers -l + ; User "WiZ" removing the limit for + the number of users on channel "#eu- + opers". + + MODE &oulu +b ; Command to list ban masks set for + the channel "&oulu". + + MODE &oulu +b *!*@* ; Command to prevent all users from + joining. + + MODE &oulu +b *!*@*.edu +e *!*@*.bu.edu + ; Command to prevent any user from a + hostname matching *.edu from joining, + except if matching *.bu.edu + + MODE #bu +be *!*@*.edu *!*@*.bu.edu + ; Comment to prevent any user from a + hostname matching *.edu from joining, + except if matching *.bu.edu + + MODE #meditation e ; Command to list exception masks set + for the channel "#meditation". + + MODE #meditation I ; Command to list invitations masks + set for the channel "#meditation". + + MODE !12345ircd O ; Command to ask who the channel + creator for "!12345ircd" is + +3.2.4 Topic message + + Command: TOPIC + Parameters: [ ] + + The TOPIC command is used to change or view the topic of a channel. + The topic for channel is returned if there is no + given. If the parameter is present, the topic for that + channel will be changed, if this action is allowed for the user + requesting it. If the parameter is an empty string, the + topic for that channel will be removed. + + + + + + +Kalt Informational [Page 19] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + Numeric Replies: + + ERR_NEEDMOREPARAMS ERR_NOTONCHANNEL + RPL_NOTOPIC RPL_TOPIC + ERR_CHANOPRIVSNEEDED ERR_NOCHANMODES + + Examples: + + :WiZ!jto@tolsun.oulu.fi TOPIC #test :New topic ; User Wiz setting the + topic. + + TOPIC #test :another topic ; Command to set the topic on #test + to "another topic". + + TOPIC #test : ; Command to clear the topic on + #test. + + TOPIC #test ; Command to check the topic for + #test. + +3.2.5 Names message + + Command: NAMES + Parameters: [ *( "," ) [ ] ] + + By using the NAMES command, a user can list all nicknames that are + visible to him. For more details on what is visible and what is not, + see "Internet Relay Chat: Channel Management" [IRC-CHAN]. The + parameter specifies which channel(s) to return information + about. There is no error reply for bad channel names. + + If no parameter is given, a list of all channels and their + occupants is returned. At the end of this list, a list of users who + are visible but either not on any channel or not on a visible channel + are listed as being on `channel' "*". + + If the parameter is specified, the request is forwarded to + that server which will generate the reply. + + Wildcards are allowed in the parameter. + + Numerics: + + ERR_TOOMANYMATCHES ERR_NOSUCHSERVER + RPL_NAMREPLY RPL_ENDOFNAMES + + + + + + +Kalt Informational [Page 20] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + Examples: + + NAMES #twilight_zone,#42 ; Command to list visible users on + #twilight_zone and #42 + + NAMES ; Command to list all visible + channels and users + +3.2.6 List message + + Command: LIST + Parameters: [ *( "," ) [ ] ] + + The list command is used to list channels and their topics. If the + parameter is used, only the status of that channel is + displayed. + + If the parameter is specified, the request is forwarded to + that server which will generate the reply. + + Wildcards are allowed in the parameter. + + Numeric Replies: + + ERR_TOOMANYMATCHES ERR_NOSUCHSERVER + RPL_LIST RPL_LISTEND + + Examples: + + LIST ; Command to list all channels. + + LIST #twilight_zone,#42 ; Command to list channels + #twilight_zone and #42 + +3.2.7 Invite message + + Command: INVITE + Parameters: + + The INVITE command is used to invite a user to a channel. The + parameter is the nickname of the person to be invited to + the target channel . There is no requirement that the + channel the target user is being invited to must exist or be a valid + channel. However, if the channel exists, only members of the channel + are allowed to invite other users. When the channel has invite-only + flag set, only channel operators may issue INVITE command. + + + + + +Kalt Informational [Page 21] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + Only the user inviting and the user being invited will receive + notification of the invitation. Other channel members are not + notified. (This is unlike the MODE changes, and is occasionally the + source of trouble for users.) + + Numeric Replies: + + ERR_NEEDMOREPARAMS ERR_NOSUCHNICK + ERR_NOTONCHANNEL ERR_USERONCHANNEL + ERR_CHANOPRIVSNEEDED + RPL_INVITING RPL_AWAY + + Examples: + + :Angel!wings@irc.org INVITE Wiz #Dust + + ; Message to WiZ when he has been + invited by user Angel to channel + #Dust + + INVITE Wiz #Twilight_Zone ; Command to invite WiZ to + #Twilight_zone + +3.2.8 Kick command + + Command: KICK + Parameters: *( "," ) *( "," ) + [] + + The KICK command can be used to request the forced removal of a user + from a channel. It causes the to PART from the by + force. For the message to be syntactically correct, there MUST be + either one channel parameter and multiple user parameter, or as many + channel parameters as there are user parameters. If a "comment" is + given, this will be sent instead of the default message, the nickname + of the user issuing the KICK. + + The server MUST NOT send KICK messages with multiple channels or + users to clients. This is necessarily to maintain backward + compatibility with old client software. + + Numeric Replies: + + ERR_NEEDMOREPARAMS ERR_NOSUCHCHANNEL + ERR_BADCHANMASK ERR_CHANOPRIVSNEEDED + ERR_USERNOTINCHANNEL ERR_NOTONCHANNEL + + + + + +Kalt Informational [Page 22] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + Examples: + + KICK &Melbourne Matthew ; Command to kick Matthew from + &Melbourne + + KICK #Finnish John :Speaking English + ; Command to kick John from #Finnish + using "Speaking English" as the + reason (comment). + + :WiZ!jto@tolsun.oulu.fi KICK #Finnish John + ; KICK message on channel #Finnish + from WiZ to remove John from channel + +3.3 Sending messages + + The main purpose of the IRC protocol is to provide a base for clients + to communicate with each other. PRIVMSG, NOTICE and SQUERY + (described in Section 3.5 on Service Query and Commands) are the only + messages available which actually perform delivery of a text message + from one client to another - the rest just make it possible and try + to ensure it happens in a reliable and structured manner. + +3.3.1 Private messages + + Command: PRIVMSG + Parameters: + + PRIVMSG is used to send private messages between users, as well as to + send messages to channels. is usually the nickname of + the recipient of the message, or a channel name. + + The parameter may also be a host mask (#) or server + mask ($). In both cases the server will only send the PRIVMSG + to those who have a server or host matching the mask. The mask MUST + have at least 1 (one) "." in it and no wildcards following the last + ".". This requirement exists to prevent people sending messages to + "#*" or "$*", which would broadcast to all users. Wildcards are the + '*' and '?' characters. This extension to the PRIVMSG command is + only available to operators. + + Numeric Replies: + + ERR_NORECIPIENT ERR_NOTEXTTOSEND + ERR_CANNOTSENDTOCHAN ERR_NOTOPLEVEL + ERR_WILDTOPLEVEL ERR_TOOMANYTARGETS + ERR_NOSUCHNICK + RPL_AWAY + + + +Kalt Informational [Page 23] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + Examples: + + :Angel!wings@irc.org PRIVMSG Wiz :Are you receiving this message ? + ; Message from Angel to Wiz. + + PRIVMSG Angel :yes I'm receiving it ! + ; Command to send a message to Angel. + + PRIVMSG jto@tolsun.oulu.fi :Hello ! + ; Command to send a message to a user + on server tolsun.oulu.fi with + username of "jto". + + PRIVMSG kalt%millennium.stealth.net@irc.stealth.net :Are you a frog? + ; Message to a user on server + irc.stealth.net with username of + "kalt", and connected from the host + millennium.stealth.net. + + PRIVMSG kalt%millennium.stealth.net :Do you like cheese? + ; Message to a user on the local + server with username of "kalt", and + connected from the host + millennium.stealth.net. + + PRIVMSG Wiz!jto@tolsun.oulu.fi :Hello ! + ; Message to the user with nickname + Wiz who is connected from the host + tolsun.oulu.fi and has the username + "jto". + + PRIVMSG $*.fi :Server tolsun.oulu.fi rebooting. + ; Message to everyone on a server + which has a name matching *.fi. + + PRIVMSG #*.edu :NSFNet is undergoing work, expect interruptions + ; Message to all users who come from + a host which has a name matching + *.edu. + +3.3.2 Notice + + Command: NOTICE + Parameters: + + The NOTICE command is used similarly to PRIVMSG. The difference + between NOTICE and PRIVMSG is that automatic replies MUST NEVER be + sent in response to a NOTICE message. This rule applies to servers + + + +Kalt Informational [Page 24] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + too - they MUST NOT send any error reply back to the client on + receipt of a notice. The object of this rule is to avoid loops + between clients automatically sending something in response to + something it received. + + This command is available to services as well as users. + + This is typically used by services, and automatons (clients with + either an AI or other interactive program controlling their actions). + + See PRIVMSG for more details on replies and examples. + +3.4 Server queries and commands + + The server query group of commands has been designed to return + information about any server which is connected to the network. + + In these queries, where a parameter appears as , wildcard + masks are usually valid. For each parameter, however, only one query + and set of replies is to be generated. In most cases, if a nickname + is given, it will mean the server to which the user is connected. + + These messages typically have little value for services, it is + therefore RECOMMENDED to forbid services from using them. + +3.4.1 Motd message + + Command: MOTD + Parameters: [ ] + + The MOTD command is used to get the "Message Of The Day" of the given + server, or current server if is omitted. + + Wildcards are allowed in the parameter. + + Numeric Replies: + RPL_MOTDSTART RPL_MOTD + RPL_ENDOFMOTD ERR_NOMOTD + +3.4.2 Lusers message + + Command: LUSERS + Parameters: [ [ ] ] + + The LUSERS command is used to get statistics about the size of the + IRC network. If no parameter is given, the reply will be about the + whole net. If a is specified, then the reply will only + + + + +Kalt Informational [Page 25] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + concern the part of the network formed by the servers matching the + mask. Finally, if the parameter is specified, the request + is forwarded to that server which will generate the reply. + + Wildcards are allowed in the parameter. + + Numeric Replies: + + RPL_LUSERCLIENT RPL_LUSEROP + RPL_LUSERUNKOWN RPL_LUSERCHANNELS + RPL_LUSERME ERR_NOSUCHSERVER + +3.4.3 Version message + + Command: VERSION + Parameters: [ ] + + The VERSION command is used to query the version of the server + program. An optional parameter is used to query the version + of the server program which a client is not directly connected to. + + Wildcards are allowed in the parameter. + + Numeric Replies: + + ERR_NOSUCHSERVER RPL_VERSION + + Examples: + + VERSION tolsun.oulu.fi ; Command to check the version of + server "tolsun.oulu.fi". + +3.4.4 Stats message + + Command: STATS + Parameters: [ [ ] ] + + The stats command is used to query statistics of certain server. If + parameter is omitted, only the end of stats reply is sent + back. + + A query may be given for any single letter which is only checked by + the destination server and is otherwise passed on by intermediate + servers, ignored and unaltered. + + Wildcards are allowed in the parameter. + + + + + +Kalt Informational [Page 26] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + Except for the ones below, the list of valid queries is + implementation dependent. The standard queries below SHOULD be + supported by the server: + + l - returns a list of the server's connections, showing how + long each connection has been established and the + traffic over that connection in Kbytes and messages for + each direction; + m - returns the usage count for each of commands supported + by the server; commands for which the usage count is + zero MAY be omitted; + o - returns a list of configured privileged users, + operators; + u - returns a string showing how long the server has been + up. + + It is also RECOMMENDED that client and server access configuration be + published this way. + + Numeric Replies: + + ERR_NOSUCHSERVER + RPL_STATSLINKINFO RPL_STATSUPTIME + RPL_STATSCOMMANDS RPL_STATSOLINE + RPL_ENDOFSTATS + + Examples: + + STATS m ; Command to check the command usage + for the server you are connected to + +3.4.5 Links message + + Command: LINKS + Parameters: [ [ ] ] + + With LINKS, a user can list all servernames, which are known by the + server answering the query. The returned list of servers MUST match + the mask, or if no mask is given, the full list is returned. + + If is given in addition to , the LINKS + command is forwarded to the first server found that matches that name + (if any), and that server is then required to answer the query. + + Numeric Replies: + + ERR_NOSUCHSERVER + RPL_LINKS RPL_ENDOFLINKS + + + +Kalt Informational [Page 27] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + Examples: + + LINKS *.au ; Command to list all servers which + have a name that matches *.au; + + LINKS *.edu *.bu.edu ; Command to list servers matching + *.bu.edu as seen by the first server + matching *.edu. + +3.4.6 Time message + + Command: TIME + Parameters: [ ] + + The time command is used to query local time from the specified + server. If the parameter is not given, the server receiving + the command must reply to the query. + + Wildcards are allowed in the parameter. + + Numeric Replies: + + ERR_NOSUCHSERVER RPL_TIME + + Examples: + TIME tolsun.oulu.fi ; check the time on the server + "tolson.oulu.fi" + +3.4.7 Connect message + + Command: CONNECT + Parameters: [ ] + + The CONNECT command can be used to request a server to try to + establish a new connection to another server immediately. CONNECT is + a privileged command and SHOULD be available only to IRC Operators. + If a is given and its mask doesn't match name of the + parsing server, the CONNECT attempt is sent to the first match of + remote server. Otherwise the CONNECT attempt is made by the server + processing the request. + + The server receiving a remote CONNECT command SHOULD generate a + WALLOPS message describing the source and target of the request. + + Numeric Replies: + + ERR_NOSUCHSERVER ERR_NOPRIVILEGES + ERR_NEEDMOREPARAMS + + + +Kalt Informational [Page 28] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + Examples: + + CONNECT tolsun.oulu.fi 6667 ; Command to attempt to connect local + server to tolsun.oulu.fi on port 6667 + +3.4.8 Trace message + + Command: TRACE + Parameters: [ ] + + TRACE command is used to find the route to specific server and + information about its peers. Each server that processes this command + MUST report to the sender about it. The replies from pass-through + links form a chain, which shows route to destination. After sending + this reply back, the query MUST be sent to the next server until + given server is reached. + + TRACE command is used to find the route to specific server. Each + server that processes this message MUST tell the sender about it by + sending a reply indicating it is a pass-through link, forming a chain + of replies. After sending this reply back, it MUST then send the + TRACE message to the next server until given server is reached. If + the parameter is omitted, it is RECOMMENDED that TRACE + command sends a message to the sender telling which servers the local + server has direct connection to. + + If the destination given by is an actual server, the + destination server is REQUIRED to report all servers, services and + operators which are connected to it; if the command was issued by an + operator, the server MAY also report all users which are connected to + it. If the destination given by is a nickname, then only a + reply for that nickname is given. If the parameter is + omitted, it is RECOMMENDED that the TRACE command is parsed as + targeted to the processing server. + + Wildcards are allowed in the parameter. + + Numeric Replies: + + ERR_NOSUCHSERVER + + If the TRACE message is destined for another server, all + intermediate servers must return a RPL_TRACELINK reply to indicate + that the TRACE passed through it and where it is going next. + + RPL_TRACELINK + + + + + +Kalt Informational [Page 29] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + A TRACE reply may be composed of any number of the following + numeric replies. + + RPL_TRACECONNECTING RPL_TRACEHANDSHAKE + RPL_TRACEUNKNOWN RPL_TRACEOPERATOR + RPL_TRACEUSER RPL_TRACESERVER + RPL_TRACESERVICE RPL_TRACENEWTYPE + RPL_TRACECLASS RPL_TRACELOG + RPL_TRACEEND + + Examples: + + TRACE *.oulu.fi ; TRACE to a server matching + *.oulu.fi + +3.4.9 Admin command + + Command: ADMIN + Parameters: [ ] + + The admin command is used to find information about the administrator + of the given server, or current server if parameter is + omitted. Each server MUST have the ability to forward ADMIN messages + to other servers. + + Wildcards are allowed in the parameter. + + Numeric Replies: + + ERR_NOSUCHSERVER + RPL_ADMINME RPL_ADMINLOC1 + RPL_ADMINLOC2 RPL_ADMINEMAIL + + Examples: + + ADMIN tolsun.oulu.fi ; request an ADMIN reply from + tolsun.oulu.fi + + ADMIN syrk ; ADMIN request for the server to + which the user syrk is connected + + + + + + + + + + + +Kalt Informational [Page 30] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + +3.4.10 Info command + + Command: INFO + Parameters: [ ] + + The INFO command is REQUIRED to return information describing the + server: its version, when it was compiled, the patchlevel, when it + was started, and any other miscellaneous information which may be + considered to be relevant. + + Wildcards are allowed in the parameter. + + Numeric Replies: + + ERR_NOSUCHSERVER + RPL_INFO RPL_ENDOFINFO + + Examples: + + INFO csd.bu.edu ; request an INFO reply from + csd.bu.edu + + INFO Angel ; request info from the server that + Angel is connected to. + +3.5 Service Query and Commands + + The service query group of commands has been designed to return + information about any service which is connected to the network. + +3.5.1 Servlist message + + Command: SERVLIST + Parameters: [ [ ] ] + + The SERVLIST command is used to list services currently connected to + the network and visible to the user issuing the command. The + optional parameters may be used to restrict the result of the query + (to matching services names, and services type). + + Numeric Replies: + + RPL_SERVLIST RPL_SERVLISTEND + + + + + + + + +Kalt Informational [Page 31] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + +3.5.2 Squery + + Command: SQUERY + Parameters: + + The SQUERY command is used similarly to PRIVMSG. The only difference + is that the recipient MUST be a service. This is the only way for a + text message to be delivered to a service. + + See PRIVMSG for more details on replies and example. + + Examples: + + SQUERY irchelp :HELP privmsg + ; Message to the service with + nickname irchelp. + + SQUERY dict@irc.fr :fr2en blaireau + ; Message to the service with name + dict@irc.fr. + +3.6 User based queries + + User queries are a group of commands which are primarily concerned + with finding details on a particular user or group users. When using + wildcards with any of these commands, if they match, they will only + return information on users who are 'visible' to you. The visibility + of a user is determined as a combination of the user's mode and the + common set of channels you are both on. + + Although services SHOULD NOT be using this class of message, they are + allowed to. + +3.6.1 Who query + + Command: WHO + Parameters: [ [ "o" ] ] + + The WHO command is used by a client to generate a query which returns + a list of information which 'matches' the parameter given by + the client. In the absence of the parameter, all visible + (users who aren't invisible (user mode +i) and who don't have a + common channel with the requesting client) are listed. The same + result can be achieved by using a of "0" or any wildcard which + will end up matching every visible user. + + The passed to WHO is matched against users' host, server, real + name and nickname if the channel cannot be found. + + + +Kalt Informational [Page 32] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + If the "o" parameter is passed only operators are returned according + to the supplied. + + Numeric Replies: + + ERR_NOSUCHSERVER + RPL_WHOREPLY RPL_ENDOFWHO + + Examples: + + WHO *.fi ; Command to list all users who match + against "*.fi". + + WHO jto* o ; Command to list all users with a + match against "jto*" if they are an + operator. + +3.6.2 Whois query + + Command: WHOIS + Parameters: [ ] *( "," ) + + This command is used to query information about particular user. + The server will answer this command with several numeric messages + indicating different statuses of each user which matches the mask (if + you are entitled to see them). If no wildcard is present in the + , any information about that nick which you are allowed to see + is presented. + + If the parameter is specified, it sends the query to a + specific server. It is useful if you want to know how long the user + in question has been idle as only local server (i.e., the server the + user is directly connected to) knows that information, while + everything else is globally known. + + Wildcards are allowed in the parameter. + + Numeric Replies: + + ERR_NOSUCHSERVER ERR_NONICKNAMEGIVEN + RPL_WHOISUSER RPL_WHOISCHANNELS + RPL_WHOISCHANNELS RPL_WHOISSERVER + RPL_AWAY RPL_WHOISOPERATOR + RPL_WHOISIDLE ERR_NOSUCHNICK + RPL_ENDOFWHOIS + + + + + + +Kalt Informational [Page 33] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + Examples: + + WHOIS wiz ; return available user information + about nick WiZ + + WHOIS eff.org trillian ; ask server eff.org for user + information about trillian + +3.6.3 Whowas + + Command: WHOWAS + Parameters: *( "," ) [ [ ] ] + + Whowas asks for information about a nickname which no longer exists. + This may either be due to a nickname change or the user leaving IRC. + In response to this query, the server searches through its nickname + history, looking for any nicks which are lexically the same (no wild + card matching here). The history is searched backward, returning the + most recent entry first. If there are multiple entries, up to + replies will be returned (or all of them if no + parameter is given). If a non-positive number is passed as being + , then a full search is done. + + Wildcards are allowed in the parameter. + + Numeric Replies: + + ERR_NONICKNAMEGIVEN ERR_WASNOSUCHNICK + RPL_WHOWASUSER RPL_WHOISSERVER + RPL_ENDOFWHOWAS + + Examples: + + WHOWAS Wiz ; return all information in the nick + history about nick "WiZ"; + + WHOWAS Mermaid 9 ; return at most, the 9 most recent + entries in the nick history for + "Mermaid"; + + WHOWAS Trillian 1 *.edu ; return the most recent history for + "Trillian" from the first server + found to match "*.edu". + +3.7 Miscellaneous messages + + Messages in this category do not fit into any of the above categories + but are nonetheless still a part of and REQUIRED by the protocol. + + + +Kalt Informational [Page 34] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + +3.7.1 Kill message + + Command: KILL + Parameters: + + The KILL command is used to cause a client-server connection to be + closed by the server which has the actual connection. Servers + generate KILL messages on nickname collisions. It MAY also be + available available to users who have the operator status. + + Clients which have automatic reconnect algorithms effectively make + this command useless since the disconnection is only brief. It does + however break the flow of data and can be used to stop large amounts + of 'flooding' from abusive users or accidents. Abusive users usually + don't care as they will reconnect promptly and resume their abusive + behaviour. To prevent this command from being abused, any user may + elect to receive KILL messages generated for others to keep an 'eye' + on would be trouble spots. + + In an arena where nicknames are REQUIRED to be globally unique at all + times, KILL messages are sent whenever 'duplicates' are detected + (that is an attempt to register two users with the same nickname) in + the hope that both of them will disappear and only 1 reappear. + + When a client is removed as the result of a KILL message, the server + SHOULD add the nickname to the list of unavailable nicknames in an + attempt to avoid clients to reuse this name immediately which is + usually the pattern of abusive behaviour often leading to useless + "KILL loops". See the "IRC Server Protocol" document [IRC-SERVER] + for more information on this procedure. + + The comment given MUST reflect the actual reason for the KILL. For + server-generated KILLs it usually is made up of details concerning + the origins of the two conflicting nicknames. For users it is left + up to them to provide an adequate reason to satisfy others who see + it. To prevent/discourage fake KILLs from being generated to hide + the identify of the KILLer, the comment also shows a 'kill-path' + which is updated by each server it passes through, each prepending + its name to the path. + + Numeric Replies: + + ERR_NOPRIVILEGES ERR_NEEDMOREPARAMS + ERR_NOSUCHNICK ERR_CANTKILLSERVER + + + + + + + +Kalt Informational [Page 35] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + NOTE: + It is RECOMMENDED that only Operators be allowed to kill other users + with KILL command. This command has been the subject of many + controversies over the years, and along with the above + recommendation, it is also widely recognized that not even operators + should be allowed to kill users on remote servers. + +3.7.2 Ping message + + Command: PING + Parameters: [ ] + + The PING command is used to test the presence of an active client or + server at the other end of the connection. Servers send a PING + message at regular intervals if no other activity detected coming + from a connection. If a connection fails to respond to a PING + message within a set amount of time, that connection is closed. A + PING message MAY be sent even if the connection is active. + + When a PING message is received, the appropriate PONG message MUST be + sent as reply to (server which sent the PING message out) + as soon as possible. If the parameter is specified, it + represents the target of the ping, and the message gets forwarded + there. + + Numeric Replies: + + ERR_NOORIGIN ERR_NOSUCHSERVER + + Examples: + + PING tolsun.oulu.fi ; Command to send a PING message to + server + + PING WiZ tolsun.oulu.fi ; Command from WiZ to send a PING + message to server "tolsun.oulu.fi" + + PING :irc.funet.fi ; Ping message sent by server + "irc.funet.fi" + + + + + + + + + + + + +Kalt Informational [Page 36] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + +3.7.3 Pong message + + Command: PONG + Parameters: [ ] + + PONG message is a reply to ping message. If parameter is + given, this message MUST be forwarded to given target. The + parameter is the name of the entity who has responded to PING message + and generated this message. + + Numeric Replies: + + ERR_NOORIGIN ERR_NOSUCHSERVER + + Example: + + PONG csd.bu.edu tolsun.oulu.fi ; PONG message from csd.bu.edu to + tolsun.oulu.fi + +3.7.4 Error + + Command: ERROR + Parameters: + + The ERROR command is for use by servers when reporting a serious or + fatal error to its peers. It may also be sent from one server to + another but MUST NOT be accepted from any normal unknown clients. + + Only an ERROR message SHOULD be used for reporting errors which occur + with a server-to-server link. An ERROR message is sent to the server + at the other end (which reports it to appropriate local users and + logs) and to appropriate local users and logs. It is not to be + passed onto any other servers by a server if it is received from a + server. + + The ERROR message is also used before terminating a client + connection. + + When a server sends a received ERROR message to its operators, the + message SHOULD be encapsulated inside a NOTICE message, indicating + that the client was not responsible for the error. + + Numerics: + + None. + + + + + + +Kalt Informational [Page 37] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + Examples: + + ERROR :Server *.fi already exists ; ERROR message to the other server + which caused this error. + + NOTICE WiZ :ERROR from csd.bu.edu -- Server *.fi already exists + ; Same ERROR message as above but + sent to user WiZ on the other server. + +4. Optional features + + This section describes OPTIONAL messages. They are not required in a + working server implementation of the protocol described herein. In + the absence of the feature, an error reply message MUST be generated + or an unknown command error. If the message is destined for another + server to answer then it MUST be passed on (elementary parsing + REQUIRED) The allocated numerics for this are listed with the + messages below. + + From this section, only the USERHOST and ISON messages are available + to services. + +4.1 Away + + Command: AWAY + Parameters: [ ] + + With the AWAY command, clients can set an automatic reply string for + any PRIVMSG commands directed at them (not to a channel they are on). + The server sends an automatic reply to the client sending the PRIVMSG + command. The only replying server is the one to which the sending + client is connected to. + + The AWAY command is used either with one parameter, to set an AWAY + message, or with no parameters, to remove the AWAY message. + + Because of its high cost (memory and bandwidth wise), the AWAY + message SHOULD only be used for client-server communication. A + server MAY choose to silently ignore AWAY messages received from + other servers. To update the away status of a client across servers, + the user mode 'a' SHOULD be used instead. (See Section 3.1.5) + + Numeric Replies: + + RPL_UNAWAY RPL_NOWAWAY + + + + + + +Kalt Informational [Page 38] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + Example: + + AWAY :Gone to lunch. Back in 5 ; Command to set away message to + "Gone to lunch. Back in 5". + +4.2 Rehash message + + Command: REHASH + Parameters: None + + The rehash command is an administrative command which can be used by + an operator to force the server to re-read and process its + configuration file. + + Numeric Replies: + + RPL_REHASHING ERR_NOPRIVILEGES + + + Example: + + REHASH ; message from user with operator + status to server asking it to reread + its configuration file. + +4.3 Die message + + Command: DIE + Parameters: None + + An operator can use the DIE command to shutdown the server. This + message is optional since it may be viewed as a risk to allow + arbitrary people to connect to a server as an operator and execute + this command. + + The DIE command MUST always be fully processed by the server to which + the sending client is connected and MUST NOT be passed onto other + connected servers. + + Numeric Replies: + + ERR_NOPRIVILEGES + + Example: + + DIE ; no parameters required. + + + + + +Kalt Informational [Page 39] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + +4.4 Restart message + + Command: RESTART + Parameters: None + + An operator can use the restart command to force the server to + restart itself. This message is optional since it may be viewed as a + risk to allow arbitrary people to connect to a server as an operator + and execute this command, causing (at least) a disruption to service. + + The RESTART command MUST always be fully processed by the server to + which the sending client is connected and MUST NOT be passed onto + other connected servers. + + Numeric Replies: + + ERR_NOPRIVILEGES + + Example: + + RESTART ; no parameters required. + +4.5 Summon message + + Command: SUMMON + Parameters: [ [ ] ] + + The SUMMON command can be used to give users who are on a host + running an IRC server a message asking them to please join IRC. This + message is only sent if the target server (a) has SUMMON enabled, (b) + the user is logged in and (c) the server process can write to the + user's tty (or similar). + + If no parameter is given it tries to summon from the + server the client is connected to is assumed as the target. + + If summon is not enabled in a server, it MUST return the + ERR_SUMMONDISABLED numeric. + + Numeric Replies: + + ERR_NORECIPIENT ERR_FILEERROR + ERR_NOLOGIN ERR_NOSUCHSERVER + ERR_SUMMONDISABLED RPL_SUMMONING + + + + + + + +Kalt Informational [Page 40] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + Examples: + + SUMMON jto ; summon user jto on the server's + host + + SUMMON jto tolsun.oulu.fi ; summon user jto on the host which a + server named "tolsun.oulu.fi" is + running. + +4.6 Users + + Command: USERS + Parameters: [ ] + + The USERS command returns a list of users logged into the server in a + format similar to the UNIX commands who(1), rusers(1) and finger(1). + If disabled, the correct numeric MUST be returned to indicate this. + + Because of the security implications of such a command, it SHOULD be + disabled by default in server implementations. Enabling it SHOULD + require recompiling the server or some equivalent change rather than + simply toggling an option and restarting the server. The procedure + to enable this command SHOULD also include suitable large comments. + + Numeric Replies: + + ERR_NOSUCHSERVER ERR_FILEERROR + RPL_USERSSTART RPL_USERS + RPL_NOUSERS RPL_ENDOFUSERS + ERR_USERSDISABLED + + Disabled Reply: + + ERR_USERSDISABLED + + Example: + + USERS eff.org ; request a list of users logged in + on server eff.org + +4.7 Operwall message + + Command: WALLOPS + Parameters: + + The WALLOPS command is used to send a message to all currently + connected users who have set the 'w' user mode for themselves. (See + Section 3.1.5 "User modes"). + + + +Kalt Informational [Page 41] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + After implementing WALLOPS as a user command it was found that it was + often and commonly abused as a means of sending a message to a lot of + people. Due to this, it is RECOMMENDED that the implementation of + WALLOPS allows and recognizes only servers as the originators of + WALLOPS. + + Numeric Replies: + + ERR_NEEDMOREPARAMS + + Example: + + :csd.bu.edu WALLOPS :Connect '*.uiuc.edu 6667' from Joshua ; WALLOPS + message from csd.bu.edu announcing a + CONNECT message it received from + Joshua and acted upon. + +4.8 Userhost message + + Command: USERHOST + Parameters: *( SPACE ) + + The USERHOST command takes a list of up to 5 nicknames, each + separated by a space character and returns a list of information + about each nickname that it found. The returned list has each reply + separated by a space. + + Numeric Replies: + + RPL_USERHOST ERR_NEEDMOREPARAMS + + Example: + + USERHOST Wiz Michael syrk ; USERHOST request for information on + nicks "Wiz", "Michael", and "syrk" + + :ircd.stealth.net 302 yournick :syrk=+syrk@millennium.stealth.net + ; Reply for user syrk + +4.9 Ison message + + Command: ISON + Parameters: *( SPACE ) + + The ISON command was implemented to provide a quick and efficient + means to get a response about whether a given nickname was currently + on IRC. ISON only takes one (1) type of parameter: a space-separated + list of nicks. For each nickname in the list that is present, the + + + +Kalt Informational [Page 42] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + server adds that to its reply string. Thus the reply string may + return empty (none of the given nicks are present), an exact copy of + the parameter string (all of them present) or any other subset of the + set of nicks given in the parameter. The only limit on the number of + nicks that may be checked is that the combined length MUST NOT be too + large as to cause the server to chop it off so it fits in 512 + characters. + + ISON is only processed by the server local to the client sending the + command and thus not passed onto other servers for further + processing. + + Numeric Replies: + + RPL_ISON ERR_NEEDMOREPARAMS + + Example: + + ISON phone trillian WiZ jarlek Avalon Angel Monstah syrk + ; Sample ISON request for 7 nicks. + +5. Replies + + The following is a list of numeric replies which are generated in + response to the commands given above. Each numeric is given with its + number, name and reply string. + +5.1 Command responses + + Numerics in the range from 001 to 099 are used for client-server + connections only and should never travel between servers. Replies + generated in the response to commands are found in the range from 200 + to 399. + + 001 RPL_WELCOME + "Welcome to the Internet Relay Network + !@" + 002 RPL_YOURHOST + "Your host is , running version " + 003 RPL_CREATED + "This server was created " + 004 RPL_MYINFO + " + " + + - The server sends Replies 001 to 004 to a user upon + successful registration. + + + + +Kalt Informational [Page 43] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + 005 RPL_BOUNCE + "Try server , port " + + - Sent by the server to a user to suggest an alternative + server. This is often used when the connection is + refused because the server is already full. + + 302 RPL_USERHOST + ":*1 *( " " )" + + - Reply format used by USERHOST to list replies to + the query list. The reply string is composed as + follows: + + reply = nickname [ "*" ] "=" ( "+" / "-" ) hostname + + The '*' indicates whether the client has registered + as an Operator. The '-' or '+' characters represent + whether the client has set an AWAY message or not + respectively. + + 303 RPL_ISON + ":*1 *( " " )" + + - Reply format used by ISON to list replies to the + query list. + + 301 RPL_AWAY + " :" + 305 RPL_UNAWAY + ":You are no longer marked as being away" + 306 RPL_NOWAWAY + ":You have been marked as being away" + + - These replies are used with the AWAY command (if + allowed). RPL_AWAY is sent to any client sending a + PRIVMSG to a client which is away. RPL_AWAY is only + sent by the server to which the client is connected. + Replies RPL_UNAWAY and RPL_NOWAWAY are sent when the + client removes and sets an AWAY message. + + 311 RPL_WHOISUSER + " * :" + 312 RPL_WHOISSERVER + " :" + 313 RPL_WHOISOPERATOR + " :is an IRC operator" + + + + +Kalt Informational [Page 44] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + 317 RPL_WHOISIDLE + " :seconds idle" + 318 RPL_ENDOFWHOIS + " :End of WHOIS list" + 319 RPL_WHOISCHANNELS + " :*( ( "@" / "+" ) " " )" + + - Replies 311 - 313, 317 - 319 are all replies + generated in response to a WHOIS message. Given that + there are enough parameters present, the answering + server MUST either formulate a reply out of the above + numerics (if the query nick is found) or return an + error reply. The '*' in RPL_WHOISUSER is there as + the literal character and not as a wild card. For + each reply set, only RPL_WHOISCHANNELS may appear + more than once (for long lists of channel names). + The '@' and '+' characters next to the channel name + indicate whether a client is a channel operator or + has been granted permission to speak on a moderated + channel. The RPL_ENDOFWHOIS reply is used to mark + the end of processing a WHOIS message. + + 314 RPL_WHOWASUSER + " * :" + 369 RPL_ENDOFWHOWAS + " :End of WHOWAS" + + - When replying to a WHOWAS message, a server MUST use + the replies RPL_WHOWASUSER, RPL_WHOISSERVER or + ERR_WASNOSUCHNICK for each nickname in the presented + list. At the end of all reply batches, there MUST + be RPL_ENDOFWHOWAS (even if there was only one reply + and it was an error). + + 321 RPL_LISTSTART + Obsolete. Not used. + + 322 RPL_LIST + " <# visible> :" + 323 RPL_LISTEND + ":End of LIST" + + - Replies RPL_LIST, RPL_LISTEND mark the actual replies + with data and end of the server's response to a LIST + command. If there are no channels available to return, + only the end reply MUST be sent. + + + + + +Kalt Informational [Page 45] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + 325 RPL_UNIQOPIS + " " + + 324 RPL_CHANNELMODEIS + " " + + 331 RPL_NOTOPIC + " :No topic is set" + 332 RPL_TOPIC + " :" + + - When sending a TOPIC message to determine the + channel topic, one of two replies is sent. If + the topic is set, RPL_TOPIC is sent back else + RPL_NOTOPIC. + + 341 RPL_INVITING + " " + + - Returned by the server to indicate that the + attempted INVITE message was successful and is + being passed onto the end client. + + 342 RPL_SUMMONING + " :Summoning user to IRC" + + - Returned by a server answering a SUMMON message to + indicate that it is summoning that user. + + 346 RPL_INVITELIST + " " + 347 RPL_ENDOFINVITELIST + " :End of channel invite list" + + - When listing the 'invitations masks' for a given channel, + a server is required to send the list back using the + RPL_INVITELIST and RPL_ENDOFINVITELIST messages. A + separate RPL_INVITELIST is sent for each active mask. + After the masks have been listed (or if none present) a + RPL_ENDOFINVITELIST MUST be sent. + + 348 RPL_EXCEPTLIST + " " + 349 RPL_ENDOFEXCEPTLIST + " :End of channel exception list" + + + + + + +Kalt Informational [Page 46] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + - When listing the 'exception masks' for a given channel, + a server is required to send the list back using the + RPL_EXCEPTLIST and RPL_ENDOFEXCEPTLIST messages. A + separate RPL_EXCEPTLIST is sent for each active mask. + After the masks have been listed (or if none present) + a RPL_ENDOFEXCEPTLIST MUST be sent. + + 351 RPL_VERSION + ". :" + + - Reply by the server showing its version details. + The is the version of the software being + used (including any patchlevel revisions) and the + is used to indicate if the server is + running in "debug mode". + + The "comments" field may contain any comments about + the version or further version details. + + 352 RPL_WHOREPLY + " + ( "H" / "G" > ["*"] [ ( "@" / "+" ) ] + : " + + 315 RPL_ENDOFWHO + " :End of WHO list" + + - The RPL_WHOREPLY and RPL_ENDOFWHO pair are used + to answer a WHO message. The RPL_WHOREPLY is only + sent if there is an appropriate match to the WHO + query. If there is a list of parameters supplied + with a WHO message, a RPL_ENDOFWHO MUST be sent + after processing each list item with being + the item. + + 353 RPL_NAMREPLY + "( "=" / "*" / "@" ) + :[ "@" / "+" ] *( " " [ "@" / "+" ] ) + - "@" is used for secret channels, "*" for private + channels, and "=" for others (public channels). + + 366 RPL_ENDOFNAMES + " :End of NAMES list" + + - To reply to a NAMES message, a reply pair consisting + of RPL_NAMREPLY and RPL_ENDOFNAMES is sent by the + server back to the client. If there is no channel + found as in the query, then only RPL_ENDOFNAMES is + + + +Kalt Informational [Page 47] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + returned. The exception to this is when a NAMES + message is sent with no parameters and all visible + channels and contents are sent back in a series of + RPL_NAMEREPLY messages with a RPL_ENDOFNAMES to mark + the end. + + 364 RPL_LINKS + " : " + 365 RPL_ENDOFLINKS + " :End of LINKS list" + + - In replying to the LINKS message, a server MUST send + replies back using the RPL_LINKS numeric and mark the + end of the list using an RPL_ENDOFLINKS reply. + + 367 RPL_BANLIST + " " + 368 RPL_ENDOFBANLIST + " :End of channel ban list" + + - When listing the active 'bans' for a given channel, + a server is required to send the list back using the + RPL_BANLIST and RPL_ENDOFBANLIST messages. A separate + RPL_BANLIST is sent for each active banmask. After the + banmasks have been listed (or if none present) a + RPL_ENDOFBANLIST MUST be sent. + + 371 RPL_INFO + ":" + 374 RPL_ENDOFINFO + ":End of INFO list" + + - A server responding to an INFO message is required to + send all its 'info' in a series of RPL_INFO messages + with a RPL_ENDOFINFO reply to indicate the end of the + replies. + + 375 RPL_MOTDSTART + ":- Message of the day - " + 372 RPL_MOTD + ":- " + 376 RPL_ENDOFMOTD + ":End of MOTD command" + + - When responding to the MOTD message and the MOTD file + is found, the file is displayed line by line, with + each line no longer than 80 characters, using + + + + +Kalt Informational [Page 48] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + RPL_MOTD format replies. These MUST be surrounded + by a RPL_MOTDSTART (before the RPL_MOTDs) and an + RPL_ENDOFMOTD (after). + + 381 RPL_YOUREOPER + ":You are now an IRC operator" + + - RPL_YOUREOPER is sent back to a client which has + just successfully issued an OPER message and gained + operator status. + + 382 RPL_REHASHING + " :Rehashing" + + - If the REHASH option is used and an operator sends + a REHASH message, an RPL_REHASHING is sent back to + the operator. + + 383 RPL_YOURESERVICE + "You are service " + + - Sent by the server to a service upon successful + registration. + + 391 RPL_TIME + " :" + + - When replying to the TIME message, a server MUST send + the reply using the RPL_TIME format above. The string + showing the time need only contain the correct day and + time there. There is no further requirement for the + time string. + + 392 RPL_USERSSTART + ":UserID Terminal Host" + 393 RPL_USERS + ": " + 394 RPL_ENDOFUSERS + ":End of users" + 395 RPL_NOUSERS + ":Nobody logged in" + + - If the USERS message is handled by a server, the + replies RPL_USERSTART, RPL_USERS, RPL_ENDOFUSERS and + RPL_NOUSERS are used. RPL_USERSSTART MUST be sent + first, following by either a sequence of RPL_USERS + or a single RPL_NOUSER. Following this is + RPL_ENDOFUSERS. + + + +Kalt Informational [Page 49] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + 200 RPL_TRACELINK + "Link + V + + " + 201 RPL_TRACECONNECTING + "Try. " + 202 RPL_TRACEHANDSHAKE + "H.S. " + 203 RPL_TRACEUNKNOWN + "???? []" + 204 RPL_TRACEOPERATOR + "Oper " + 205 RPL_TRACEUSER + "User " + 206 RPL_TRACESERVER + "Serv S C + @ V" + 207 RPL_TRACESERVICE + "Service " + 208 RPL_TRACENEWTYPE + " 0 " + 209 RPL_TRACECLASS + "Class " + 210 RPL_TRACERECONNECT + Unused. + 261 RPL_TRACELOG + "File " + 262 RPL_TRACEEND + " :End of TRACE" + + - The RPL_TRACE* are all returned by the server in + response to the TRACE message. How many are + returned is dependent on the TRACE message and + whether it was sent by an operator or not. There + is no predefined order for which occurs first. + Replies RPL_TRACEUNKNOWN, RPL_TRACECONNECTING and + RPL_TRACEHANDSHAKE are all used for connections + which have not been fully established and are either + unknown, still attempting to connect or in the + process of completing the 'server handshake'. + RPL_TRACELINK is sent by any server which handles + a TRACE message and has to pass it on to another + server. The list of RPL_TRACELINKs sent in + response to a TRACE command traversing the IRC + network should reflect the actual connectivity of + the servers themselves along that path. + + + + +Kalt Informational [Page 50] + +RFC 2812 Internet Relay Chat: Client Protocol April 2000 + + + RPL_TRACENEWTYPE is to be used for any connection + which does not fit in the other categories but is + being displayed anyway. + RPL_TRACEEND is sent to indicate the end of the list. + + 211 RPL_STATSLINKINFO + " + +