2016-09-29 13:59:51.052640 I | torus: creating metadata service: etcd 2016-09-29 13:59:51.079857 I | torus: 5fe8b368-8552-11e6-9bc9-001a4a40dc3e: 1 2016-09-29 13:59:51.079926 I | torus: 69d0e1d4-8552-11e6-aff4-001a4a40dc3d: 1 2016-09-29 13:59:51.080226 I | torus: creating blockstore: mfile 2016-09-29 13:59:51.080533 I | storage: loading block index... 2016-09-29 13:59:51.173276 D | storage: index memory usage: 626K 2016-09-29 13:59:51.173391 I | storage: done loading block index 2016-09-29 13:59:51.175190 I | torus: 5fe8b368-8552-11e6-9bc9-001a4a40dc3e: 1 2016-09-29 13:59:51.175224 I | torus: 69d0e1d4-8552-11e6-aff4-001a4a40dc3d: 1 2016-09-29 13:59:51.176332 D | gc: creating blockvol gc 2016-09-29 13:59:51.511851 D | distributor: failed local peer (again) 2016-09-29 14:01:05.182411 D | storage: mfile: block already existsbr 10000000003 : 65 : 3 2016-09-29 14:01:33.938673 D | storage: mfile: block already existsbr 3 : 66 : 11 2016-09-29 14:08:35.706742 D | storage: mfile: block already existsbr 10000000003 : 72 : 4 2016-09-29 14:09:35.813061 D | storage: mfile: block already existsbr 3 : 76 : 1a6 2016-09-29 14:09:37.968361 D | storage: mfile: block already existsbr 3 : 77 : 1d3 2016-09-29 14:26:37.400964 D | storage: mfile: block already existsbr 10000000003 : 114 : 4 2016-09-29 14:27:01.914043 D | storage: mfile: block already existsbr 10000000003 : 1b2 : 2 2016-09-29 14:27:20.320740 D | storage: mfile: block already existsbr 10000000003 : 227 : 4 2016-09-29 14:27:54.178544 D | storage: mfile: block already existsbr 10000000003 : 302 : 1 2016-09-29 14:29:27.959422 D | storage: mfile: block already existsbr 10000000003 : 53b : 4 2016-09-29 14:54:17.777866 D | storage: mfile: block already existsbr 10000000003 : b32 : 2 unexpected fault address 0x7f4fb9854000 fatal error: fault [signal 0x7 code=0x2 addr=0x7f4fb9854000 pc=0x463d11] goroutine 63169 [running]: runtime.throw(0xd168e0, 0x5) /usr/lib/golang/src/runtime/panic.go:547 +0x90 fp=0xc82033d3b0 sp=0xc82033d398 runtime.sigpanic() /usr/lib/golang/src/runtime/sigpanic_unix.go:21 +0x10c fp=0xc82033d400 sp=0xc82033d3b0 runtime.memmove(0x7f4fb9800000, 0xc8206e6000, 0x80000) /usr/lib/golang/src/runtime/memmove_amd64.s:83 +0x91 fp=0xc82033d408 sp=0xc82033d400 github.com/coreos/torus/storage.(*MFile).WriteBlock(0xc82027cc30, 0x3530, 0xc8206e6000, 0x80000, 0x80000, 0x0, 0x0) /root/work/src/github.com/coreos/torus/storage/mmap_file.go:87 +0x1d4 fp=0xc82033d510 sp=0xc82033d408 github.com/coreos/torus/storage.(*mfileBlock).WriteBlock(0xc8202686c0, 0x7fcb19643648, 0xc823fb2ab0, 0x10000000003, 0xc18, 0x2, 0xc8206e6000, 0x80000, 0x80000, 0x0, ...) /root/work/src/github.com/coreos/torus/storage/mfile.go:226 +0x4f4 fp=0xc82033d7a0 sp=0xc82033d510 github.com/coreos/torus/distributor.(*Distributor).PutBlock(0xc8202687e0, 0x7fcb19643648, 0xc823fb2ab0, 0x10000000003, 0xc18, 0x2, 0xc8206e6000, 0x80000, 0x80000, 0x0, ...) /root/work/src/github.com/coreos/torus/distributor/rpc.go:42 +0x35d fp=0xc82033d908 sp=0xc82033d7a0 github.com/coreos/torus/distributor/protocols/grpc.(*handler).PutBlock(0xc820314040, 0x7fcb19643648, 0xc823fb2ab0, 0xc82077e4b0, 0xc821e2e000, 0x0, 0x0) /root/work/src/github.com/coreos/torus/distributor/protocols/grpc/grpc.go:123 +0x188 fp=0xc82033da00 sp=0xc82033d908 github.com/coreos/torus/models._TorusStorage_PutBlock_Handler(0xc5ff20, 0xc820314040, 0x7fcb19643648, 0xc823fb2ab0, 0xc820775f40, 0x0, 0x0, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/models/rpc.pb.go:652 +0x168 fp=0xc82033daa8 sp=0xc82033da00 github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc820268870, 0x7fcb19650480, 0xc820076000, 0xc8224a6000, 0xc820314100, 0x117a1b8, 0xc823fb2a50, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:524 +0xe24 fp=0xc82033ddf0 sp=0xc82033daa8 github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).handleStream(0xc820268870, 0x7fcb19650480, 0xc820076000, 0xc8224a6000, 0xc823fb2a50) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:684 +0x109d fp=0xc8203 sp=0xc82033ddf0 github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc820248b50, 0xc820268870, 0x7fcb19650480, 0xc820076000, 0xc8224a6000) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:350 +0xa0 fp=0xc82033df78 sp=0xc82033df48 runtime.goexit() /usr/lib/golang/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc82033df80 sp=0xc82033df78 created by github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:351 +0x9a goroutine 1 [chan receive, 54 minutes]: main.runServer(0x117d340, 0xc820226740, 0x0, 0x4) /root/work/src/github.com/coreos/torus/cmd/torusd/main.go:225 +0x941 github.com/coreos/torus/vendor/github.com/spf13/cobra.(*Command).execute(0x117d340, 0xc82000a060, 0x4, 0x4, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/github.com/spf13/cobra/command.go:565 +0x85a github.com/coreos/torus/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0x117d340, 0x117d340, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/github.com/spf13/cobra/command.go:651 +0x55c github.com/coreos/torus/vendor/github.com/spf13/cobra.(*Command).Execute(0x117d340, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/github.com/spf13/cobra/command.go:610 +0x2d main.main() /root/work/src/github.com/coreos/torus/cmd/torusd/main.go:75 +0x2f goroutine 17 [syscall, 54 minutes, locked to thread]: runtime.goexit() /usr/lib/golang/src/runtime/asm_amd64.s:1998 +0x1 goroutine 5 [syscall, 54 minutes]: os/signal.signal_recv(0x0) /usr/lib/golang/src/runtime/sigqueue.go:116 +0x132 os/signal.loop() /usr/lib/golang/src/os/signal/signal_unix.go:22 +0x18 created by os/signal.init.1 /usr/lib/golang/src/os/signal/signal_unix.go:28 +0x37 goroutine 54 [select]: github.com/coreos/torus/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc8201c4960) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http2_client.go:869 +0x5da created by github.com/coreos/torus/vendor/google.golang.org/grpc/transport.newHTTP2Client /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http2_client.go:201 +0x15c2 goroutine 53 [IO wait]: net.runtime_pollWait(0x7fcb1963c298, 0x72, 0xc820254000) /usr/lib/golang/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc820235950, 0x72, 0x0, 0x0) /usr/lib/golang/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc820235950, 0x0, 0x0) /usr/lib/golang/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc8202358f0, 0xc820254000, 0x8000, 0x8000, 0x0, 0x7fcb19637050, 0xc8200121c0) /usr/lib/golang/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc820026620, 0xc820254000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/lib/golang/src/net/net.go:172 +0xe4 bufio.(*Reader).fill(0xc8201f6ea0) /usr/lib/golang/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc8201f6ea0, 0xc8201040f8, 0x9, 0x9, 0xc81fee9e87, 0x0, 0x0) /usr/lib/golang/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fcb19643098, 0xc8201f6ea0, 0xc8201040f8, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /usr/lib/golang/src/io/io.go:297 +0xe6 io.ReadFull(0x7fcb19643098, 0xc8201f6ea0, 0xc8201040f8, 0x9, 0x9, 0xc8222c2a58, 0x0, 0x0) /usr/lib/golang/src/io/io.go:315 +0x62 github.com/coreos/torus/vendor/golang.org/x/net/http2.readFrameHeader(0xc8201040f8, 0x9, 0x9, 0x7fcb19643098, 0xc8201f6ea0, 0x0, 0xc800000000, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/golang.org/x/net/http2/frame.go:237 +0xa5 github.com/coreos/torus/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc8201040c0, 0x0, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/golang.org/x/net/http2/frame.go:464 +0x106 github.com/coreos/torus/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc8202337d0, 0x0, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http_util.go:406 +0x3d github.com/coreos/torus/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc8201c4960) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http2_client.go:791 +0x109 created by github.com/coreos/torus/vendor/google.golang.org/grpc/transport.newHTTP2Client /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http2_client.go:174 +0xd21 goroutine 55 [select, 54 minutes]: github.com/coreos/torus/vendor/google.golang.org/grpc.(*Conn).transportMonitor(0xc8201c4870) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/clientconn.go:547 +0x1d3 created by github.com/coreos/torus/vendor/google.golang.org/grpc.NewConn /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/clientconn.go:346 +0x49f goroutine 56 [select, 54 minutes]: github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3.(*Client).connMonitor(0xc82024a160) /root/work/src/github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3/client.go:352 +0x452 created by github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3.newClient /root/work/src/github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3/client.go:259 +0x6b3 goroutine 57 [select, 54 minutes]: github.com/coreos/torus/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc820233dd0, 0xc82023d630, 0x5, 0x5, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/transport.go:141 +0x7e6 github.com/coreos/torus/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc820093340, 0xc82023d630, 0x5, 0x5, 0x7fcb19643800, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/transport.go:294 +0x71 io.ReadAtLeast(0x7fcb196437d8, 0xc820093340, 0xc82023d630, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) /usr/lib/golang/src/io/io.go:297 +0xe6 io.ReadFull(0x7fcb196437d8, 0xc820093340, 0xc82023d630, 0x5, 0x5, 0x7fcb19643850, 0x0, 0x0) /usr/lib/golang/src/io/io.go:315 +0x62 github.com/coreos/torus/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc82023d620, 0xd4ac10, 0x0, 0x0, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/rpc_util.go:216 +0xb9 github.com/coreos/torus/vendor/google.golang.org/grpc.recv(0xc82023d620, 0x7fcb19642e30, 0x11a2f20, 0xc820093340, 0x0, 0x0, 0xcb4060, 0xc82023d640, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/rpc_util.go:297 +0x45 github.com/coreos/torus/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc820244be0, 0xcb4060, 0xc82023d640, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/stream.go:234 +0xac github.com/coreos/torus/vendor/github.com/coreos/etcd/etcdserver/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv(0xc820248300, 0x7fcb19643850, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/github.com/coreos/etcd/etcdserver/etcdserverpb/rpc.pb.go:2365 +0x7e github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3.(*lessor).recvKeepAliveLoop(0xc8201f70e0) /root/work/src/github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3/lease.go:293 +0x96 created by github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3.NewLease /root/work/src/github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3/lease.go:111 +0x23e goroutine 58 [select, 54 minutes]: github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3.(*watcher).run(0xc8201bbc00) /root/work/src/github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3/watch.go:296 +0xa72 created by github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3.NewWatcher /root/work/src/github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3/watch.go:159 +0x363 goroutine 67 [select, 54 minutes]: github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3.(*watcher).serveStream(0xc8201bbc00, 0xc820276150) /root/work/src/github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3/watch.go:417 +0x950 created by github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3.(*watcher).addStream /root/work/src/github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3/watch.go:258 +0x310 goroutine 61 [select, 54 minutes]: github.com/coreos/torus/vendor/google.golang.org/grpc.NewClientStream.func1(0x7fcb196430e8, 0xc8201c4960, 0xc820093340, 0xc820244be0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/stream.go:151 +0x258 created by github.com/coreos/torus/vendor/google.golang.org/grpc.NewClientStream /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/stream.go:159 +0xab2 goroutine 62 [select]: github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3.(*lessor).sendKeepAliveLoop(0xc8201f70e0, 0x7fcb19643850, 0xc820248300) /root/work/src/github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3/lease.go:352 +0x514 created by github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3.(*lessor).resetRecv /root/work/src/github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3/lease.go:311 +0xcc goroutine 66 [chan receive, 54 minutes]: github.com/coreos/torus/metadata/etcd.(*Etcd).watchRing(0xc820093180, 0x7fcb19643c58, 0xc82027ca20) /root/work/src/github.com/coreos/torus/metadata/etcd/ring_watch.go:25 +0x261 created by github.com/coreos/torus/metadata/etcd.(*Etcd).watchRingUpdates /root/work/src/github.com/coreos/torus/metadata/etcd/ring_watch.go:16 +0x19d goroutine 64 [select, 54 minutes]: github.com/coreos/torus/vendor/google.golang.org/grpc.NewClientStream.func1(0x7fcb196430e8, 0xc8201c4960, 0xc820093420, 0xc820244d20) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/stream.go:151 +0x258 created by github.com/coreos/torus/vendor/google.golang.org/grpc.NewClientStream /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/stream.go:159 +0xab2 goroutine 65 [select, 54 minutes]: github.com/coreos/torus/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc820233ef0, 0xc82023d730, 0x5, 0x5, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/transport.go:141 +0x7e6 github.com/coreos/torus/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc820093420, 0xc82023d730, 0x5, 0x5, 0x9, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/transport.go:294 +0x71 io.ReadAtLeast(0x7fcb196437d8, 0xc820093420, 0xc82023d730, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) /usr/lib/golang/src/io/io.go:297 +0xe6 io.ReadFull(0x7fcb196437d8, 0xc820093420, 0xc82023d730, 0x5, 0x5, 0xc820033c70, 0x0, 0x0) /usr/lib/golang/src/io/io.go:315 +0x62 github.com/coreos/torus/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc82023d720, 0x412074, 0x0, 0x0, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/rpc_util.go:216 +0xb9 github.com/coreos/torus/vendor/google.golang.org/grpc.recv(0xc82023d720, 0x7fcb19642e30, 0x11a2f20, 0xc820093420, 0x0, 0x0, 0xcc5120, 0xc820275600, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/rpc_util.go:297 +0x45 github.com/coreos/torus/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc820244d20, 0xcc5120, 0xc820275600, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/stream.go:234 +0xac github.com/coreos/torus/vendor/github.com/coreos/etcd/etcdserver/etcdserverpb.(*watchWatchClient).Recv(0xc820248360, 0xc8201f7320, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/github.com/coreos/etcd/etcdserver/etcdserverpb/rpc.pb.go:2239 +0x7e github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3.(*watcher).serveWatchClient(0xc8201bbc00, 0x7fcb196438b0, 0xc820248360) /root/work/src/github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3/watch.go:387 +0x39 created by github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3.(*watcher).newWatchClient /root/work/src/github.com/coreos/torus/vendor/github.com/coreos/etcd/clientv3/watch.go:484 +0xb2 goroutine 68 [select, 54 minutes, locked to thread]: runtime.gopark(0xe97e08, 0xc820025f28, 0xd19f40, 0x6, 0x7bb518, 0x2) /usr/lib/golang/src/runtime/proc.go:262 +0x163 runtime.selectgoImpl(0xc820025f28, 0x0, 0x18) /usr/lib/golang/src/runtime/select.go:392 +0xa67 runtime.selectgo(0xc820025f28) /usr/lib/golang/src/runtime/select.go:215 +0x12 runtime.ensureSigM.func1() /usr/lib/golang/src/runtime/signal1_unix.go:279 +0x358 runtime.goexit() /usr/lib/golang/src/runtime/asm_amd64.s:1998 +0x1 goroutine 69 [IO wait, 53 minutes]: net.runtime_pollWait(0x7fcb1963c1d8, 0x72, 0x0) /usr/lib/golang/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc820276220, 0x72, 0x0, 0x0) /usr/lib/golang/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc820276220, 0x0, 0x0) /usr/lib/golang/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).accept(0xc8202761c0, 0x0, 0x7fcb19643fb0, 0xc82023e180) /usr/lib/golang/src/net/fd_unix.go:426 +0x27c net.(*TCPListener).AcceptTCP(0xc820026718, 0xc820108f00, 0x0, 0x0) /usr/lib/golang/src/net/tcpsock_posix.go:254 +0x4d net.(*TCPListener).Accept(0xc820026718, 0x0, 0x0, 0x0, 0x0) /usr/lib/golang/src/net/tcpsock_posix.go:264 +0x3d github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).Serve(0xc820268870, 0x7fcb19643f18, 0xc820026718, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:279 +0x1cf created by github.com/coreos/torus/distributor/protocols/grpc.grpcRPCListener /root/work/src/github.com/coreos/torus/distributor/protocols/grpc/grpc.go:39 +0x265 goroutine 70 [select, 54 minutes]: github.com/coreos/torus/distributor.(*Distributor).ringWatcher(0xc8202687e0, 0x0) /root/work/src/github.com/coreos/torus/distributor/rebalance.go:19 +0x34f created by github.com/coreos/torus/distributor.newDistributor /root/work/src/github.com/coreos/torus/distributor/distributor.go:67 +0x54a goroutine 71 [sleep]: time.Sleep(0x77359400) /usr/lib/golang/src/runtime/time.go:59 +0xf9 github.com/coreos/torus/distributor.(*Distributor).rebalanceTicker(0xc8202687e0, 0xc82031bd40) /root/work/src/github.com/coreos/torus/distributor/rebalance.go:96 +0x738 created by github.com/coreos/torus/distributor.newDistributor /root/work/src/github.com/coreos/torus/distributor/distributor.go:72 +0x8fe goroutine 79 [IO wait]: net.runtime_pollWait(0x7fcb1963c118, 0x72, 0xc82034a000) /usr/lib/golang/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc820276990, 0x72, 0x0, 0x0) /usr/lib/golang/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc820276990, 0x0, 0x0) /usr/lib/golang/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc820276930, 0xc82034a000, 0x8000, 0x8000, 0x0, 0x7fcb19637050, 0xc8200121c0) /usr/lib/golang/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc820026818, 0xc82034a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/lib/golang/src/net/net.go:172 +0xe4 bufio.(*Reader).fill(0xc820333020) /usr/lib/golang/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc820333020, 0xc820104c38, 0x9, 0x9, 0xc81fe026ed, 0x0, 0x0) /usr/lib/golang/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fcb19643098, 0xc820333020, 0xc820104c38, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /usr/lib/golang/src/io/io.go:297 +0xe6 io.ReadFull(0x7fcb19643098, 0xc820333020, 0xc820104c38, 0x9, 0x9, 0xc823fb2b48, 0x0, 0x0) /usr/lib/golang/src/io/io.go:315 +0x62 github.com/coreos/torus/vendor/golang.org/x/net/http2.readFrameHeader(0xc820104c38, 0x9, 0x9, 0x7fcb19643098, 0xc820333020, 0x0, 0xc800000000, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/golang.org/x/net/http2/frame.go:237 +0xa5 github.com/coreos/torus/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc820104c00, 0x0, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/golang.org/x/net/http2/frame.go:464 +0x106 github.com/coreos/torus/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc820335fb0, 0x0, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http_util.go:406 +0x3d github.com/coreos/torus/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc8201c4c30) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http2_client.go:791 +0x109 created by github.com/coreos/torus/vendor/google.golang.org/grpc/transport.newHTTP2Client /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http2_client.go:174 +0xd21 goroutine 73 [select]: github.com/coreos/torus.(*Server).heartbeat(0xc8202cfe60, 0xc8203321e0) /root/work/src/github.com/coreos/torus/heartbeat.go:85 +0x1d5 created by github.com/coreos/torus.(*Server).BeginHeartbeat /root/work/src/github.com/coreos/torus/heartbeat.go:77 +0x8ae goroutine 74 [chan receive, 54 minutes]: main.runServer.func1(0xc82027bb60, 0xc82027bb00) /root/work/src/github.com/coreos/torus/cmd/torusd/main.go:210 +0x64 created by main.runServer /root/work/src/github.com/coreos/torus/cmd/torusd/main.go:215 +0x778 goroutine 77 [select, 54 minutes]: github.com/coreos/torus/vendor/google.golang.org/grpc.(*Conn).transportMonitor(0xc8201c4b40) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/clientconn.go:547 +0x1d3 github.com/coreos/torus/vendor/google.golang.org/grpc.NewConn.func1(0xc8201c4b40) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/clientconn.go:355 +0x1b5 created by github.com/coreos/torus/vendor/google.golang.org/grpc.NewConn /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/clientconn.go:356 +0x4e3 goroutine 80 [select]: github.com/coreos/torus/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc8201c4c30) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http2_client.go:869 +0x5da created by github.com/coreos/torus/vendor/google.golang.org/grpc/transport.newHTTP2Client /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http2_client.go:201 +0x15c2 goroutine 82 [IO wait]: net.runtime_pollWait(0x7fcb1963c058, 0x72, 0xc8200ae000) /usr/lib/golang/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc820234140, 0x72, 0x0, 0x0) /usr/lib/golang/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc820234140, 0x0, 0x0) /usr/lib/golang/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc8202340e0, 0xc8200ae000, 0x8000, 0x8000, 0x0, 0x7fcb19637050, 0xc8200121c0) /usr/lib/golang/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc820026020, 0xc8200ae000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/lib/golang/src/net/net.go:172 +0xe4 bufio.(*Reader).fill(0xc82027aea0) /usr/lib/golang/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc82027aea0, 0xc8201055f8, 0x9, 0x9, 0x0, 0x0, 0x0) /usr/lib/golang/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fcb19643098, 0xc82027aea0, 0xc8201055f8, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /usr/lib/golang/src/io/io.go:297 +0xe6 io.ReadFull(0x7fcb19643098, 0xc82027aea0, 0xc8201055f8, 0x9, 0x9, 0xc820001c80, 0x0, 0x0) /usr/lib/golang/src/io/io.go:315 +0x62 github.com/coreos/torus/vendor/golang.org/x/net/http2.readFrameHeader(0xc8201055f8, 0x9, 0x9, 0x7fcb19643098, 0xc82027aea0, 0x20000000, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/golang.org/x/net/http2/frame.go:237 +0xa5 github.com/coreos/torus/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc8201055c0, 0x0, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/golang.org/x/net/http2/frame.go:464 +0x106 github.com/coreos/torus/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc820334690, 0x0, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http_util.go:406 +0x3d github.com/coreos/torus/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc820076090, 0xc820334750) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http2_server.go:243 +0x63a github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc820268870, 0x7fcb19650480, 0xc820076090) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:352 +0x159 github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc820268870, 0x7fcb19642f78, 0xc820026020, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:339 +0x49d github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc820268870, 0x7fcb19642f78, 0xc820026020) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:316 +0x4ee created by github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).Serve /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:288 +0x38c goroutine 83 [select]: github.com/coreos/torus/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc820076090) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http2_server.go:652 +0x5da created by github.com/coreos/torus/vendor/google.golang.org/grpc/transport.newHTTP2Server /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http2_server.go:134 +0x84f goroutine 254 [select]: github.com/coreos/torus/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc820076000) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http2_server.go:652 +0x5da created by github.com/coreos/torus/vendor/google.golang.org/grpc/transport.newHTTP2Server /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http2_server.go:134 +0x84f goroutine 253 [IO wait]: net.runtime_pollWait(0x7fcb1963bf98, 0x72, 0xc8202ba000) /usr/lib/golang/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc8202344c0, 0x72, 0x0, 0x0) /usr/lib/golang/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc8202344c0, 0x0, 0x0) /usr/lib/golang/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc820234460, 0xc8202ba000, 0x8000, 0x8000, 0x0, 0x7fcb19637050, 0xc8200121c0) /usr/lib/golang/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc8200261e8, 0xc8202ba000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/lib/golang/src/net/net.go:172 +0xe4 bufio.(*Reader).fill(0xc82031b4a0) /usr/lib/golang/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc82031b4a0, 0xc8203e0638, 0x9, 0x9, 0x0, 0x0, 0x0) /usr/lib/golang/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fcb19643098, 0xc82031b4a0, 0xc8203e0638, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /usr/lib/golang/src/io/io.go:297 +0xe6 io.ReadFull(0x7fcb19643098, 0xc82031b4a0, 0xc8203e0638, 0x9, 0x9, 0xc820338a80, 0x0, 0x0) /usr/lib/golang/src/io/io.go:315 +0x62 github.com/coreos/torus/vendor/golang.org/x/net/http2.readFrameHeader(0xc8203e0638, 0x9, 0x9, 0x7fcb19643098, 0xc82031b4a0, 0x22000000, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/golang.org/x/net/http2/frame.go:237 +0xa5 github.com/coreos/torus/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc8203e0600, 0x0, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/golang.org/x/net/http2/frame.go:464 +0x106 github.com/coreos/torus/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc820334cc0, 0x0, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http_util.go:406 +0x3d github.com/coreos/torus/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc820076000, 0xc820334de0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/transport/http2_server.go:243 +0x63a github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc820268870, 0x7fcb19650480, 0xc820076000) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:352 +0x159 github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc820268870, 0x7fcb19642f78, 0xc8200261e8, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:339 +0x49d github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc820268870, 0x7fcb19642f78, 0xc8200261e8) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:316 +0x4ee created by github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).Serve /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:288 +0x38c goroutine 63172 [semacquire]: sync.runtime_Semacquire(0xc8202686cc) /usr/lib/golang/src/runtime/sema.go:47 +0x26 sync.(*RWMutex).RLock(0xc8202686c0) /usr/lib/golang/src/sync/rwmutex.go:37 +0x58 github.com/coreos/torus/storage.(*mfileBlock).HasBlock(0xc8202686c0, 0x7fcb19643648, 0xc82077e7b0, 0x10000000003, 0xac3, 0x4, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/storage/mfile.go:186 +0x41 github.com/coreos/torus/distributor.(*Distributor).RebalanceCheck(0xc8202687e0, 0x7fcb19643648, 0xc82077e7b0, 0xc821eb4a00, 0x32, 0x32, 0x0, 0x0, 0x0, 0x0, ...) /root/work/src/github.com/coreos/torus/distributor/rpc.go:55 +0x187 github.com/coreos/torus/distributor/protocols/grpc.(*handler).RebalanceCheck(0xc820314040, 0x7fcb19643648, 0xc82077e7b0, 0xc8224a55c0, 0xc82171c000, 0x0, 0x0) /root/work/src/github.com/coreos/torus/distributor/protocols/grpc/grpc.go:136 +0x15e github.com/coreos/torus/models._TorusStorage_RebalanceCheck_Handler(0xc5ff20, 0xc820314040, 0x7fcb19643648, 0xc82077e7b0, 0xc822a4e000, 0x0, 0x0, 0x0, 0x0, 0x0) /root/work/src/github.com/coreos/torus/models/rpc.pb.go:670 +0x168 github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc820268870, 0x7fcb19650480, 0xc820076090, 0xc8224a62a0, 0xc820314100, 0x117a1d0, 0xc82077e780, 0x0, 0x0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:524 +0xe24 github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).handleStream(0xc820268870, 0x7fcb19650480, 0xc820076090, 0xc8224a62a0, 0xc82077e780) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:684 +0x109d github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc8202487d0, 0xc820268870, 0x7fcb19650480, 0xc820076090, 0xc8224a62a0) /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:350 +0xa0 created by github.com/coreos/torus/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /root/work/src/github.com/coreos/torus/vendor/google.golang.org/grpc/server.go:351 +0x9a