binary.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634
  1. package common
  2. // Copyright 2009 The Go Authors. All rights reserved.
  3. // Use of this source code is governed by a BSD-style
  4. // license that can be found in the LICENSE file.
  5. // Package binary implements simple translation between numbers and byte
  6. // sequences and encoding and decoding of varints.
  7. //
  8. // Numbers are translated by reading and writing fixed-size values.
  9. // A fixed-size value is either a fixed-size arithmetic
  10. // type (int8, uint8, int16, float32, complex64, ...)
  11. // or an array or struct containing only fixed-size values.
  12. //
  13. // The varint functions encode and decode single integer values using
  14. // a variable-length encoding; smaller values require fewer bytes.
  15. // For a specification, see
  16. // http://code.google.com/apis/protocolbuffers/docs/encoding.html.
  17. //
  18. // This package favors simplicity over efficiency. Clients that require
  19. // high-performance serialization, especially for large data structures,
  20. // should look at more advanced solutions such as the encoding/gob
  21. // package or protocol buffers.
  22. import (
  23. "errors"
  24. "io"
  25. "math"
  26. "reflect"
  27. )
  28. // A ByteOrder specifies how to convert byte sequences into
  29. // 16-, 32-, or 64-bit unsigned integers.
  30. type ByteOrder interface {
  31. Uint16([]byte) uint16
  32. Uint32([]byte) uint32
  33. Uint64([]byte) uint64
  34. PutUint16([]byte, uint16)
  35. PutUint32([]byte, uint32)
  36. PutUint64([]byte, uint64)
  37. String() string
  38. }
  39. // LittleEndian is the little-endian implementation of ByteOrder.
  40. var LittleEndian littleEndian
  41. // BigEndian is the big-endian implementation of ByteOrder.
  42. var BigEndian bigEndian
  43. type littleEndian struct{}
  44. func (littleEndian) Uint16(b []byte) uint16 { return uint16(b[0]) | uint16(b[1])<<8 }
  45. func (littleEndian) PutUint16(b []byte, v uint16) {
  46. b[0] = byte(v)
  47. b[1] = byte(v >> 8)
  48. }
  49. func (littleEndian) Uint32(b []byte) uint32 {
  50. return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
  51. }
  52. func (littleEndian) PutUint32(b []byte, v uint32) {
  53. b[0] = byte(v)
  54. b[1] = byte(v >> 8)
  55. b[2] = byte(v >> 16)
  56. b[3] = byte(v >> 24)
  57. }
  58. func (littleEndian) Uint64(b []byte) uint64 {
  59. return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
  60. uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
  61. }
  62. func (littleEndian) PutUint64(b []byte, v uint64) {
  63. b[0] = byte(v)
  64. b[1] = byte(v >> 8)
  65. b[2] = byte(v >> 16)
  66. b[3] = byte(v >> 24)
  67. b[4] = byte(v >> 32)
  68. b[5] = byte(v >> 40)
  69. b[6] = byte(v >> 48)
  70. b[7] = byte(v >> 56)
  71. }
  72. func (littleEndian) String() string { return "LittleEndian" }
  73. func (littleEndian) GoString() string { return "binary.LittleEndian" }
  74. type bigEndian struct{}
  75. func (bigEndian) Uint16(b []byte) uint16 { return uint16(b[1]) | uint16(b[0])<<8 }
  76. func (bigEndian) PutUint16(b []byte, v uint16) {
  77. b[0] = byte(v >> 8)
  78. b[1] = byte(v)
  79. }
  80. func (bigEndian) Uint32(b []byte) uint32 {
  81. return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24
  82. }
  83. func (bigEndian) PutUint32(b []byte, v uint32) {
  84. b[0] = byte(v >> 24)
  85. b[1] = byte(v >> 16)
  86. b[2] = byte(v >> 8)
  87. b[3] = byte(v)
  88. }
  89. func (bigEndian) Uint64(b []byte) uint64 {
  90. return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 |
  91. uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56
  92. }
  93. func (bigEndian) PutUint64(b []byte, v uint64) {
  94. b[0] = byte(v >> 56)
  95. b[1] = byte(v >> 48)
  96. b[2] = byte(v >> 40)
  97. b[3] = byte(v >> 32)
  98. b[4] = byte(v >> 24)
  99. b[5] = byte(v >> 16)
  100. b[6] = byte(v >> 8)
  101. b[7] = byte(v)
  102. }
  103. func (bigEndian) String() string { return "BigEndian" }
  104. func (bigEndian) GoString() string { return "binary.BigEndian" }
  105. // Read reads structured binary data from r into data.
  106. // Data must be a pointer to a fixed-size value or a slice
  107. // of fixed-size values.
  108. // Bytes read from r are decoded using the specified byte order
  109. // and written to successive fields of the data.
  110. // When reading into structs, the field data for fields with
  111. // blank (_) field names is skipped; i.e., blank field names
  112. // may be used for padding.
  113. // When reading into a struct, all non-blank fields must be exported.
  114. func Read(r io.Reader, order ByteOrder, data interface{}) error {
  115. // Fast path for basic types and slices.
  116. if n := intDataSize(data); n != 0 {
  117. var b [8]byte
  118. var bs []byte
  119. if n > len(b) {
  120. bs = make([]byte, n)
  121. } else {
  122. bs = b[:n]
  123. }
  124. if _, err := io.ReadFull(r, bs); err != nil {
  125. return err
  126. }
  127. switch data := data.(type) {
  128. case *int8:
  129. *data = int8(b[0])
  130. case *uint8:
  131. *data = b[0]
  132. case *int16:
  133. *data = int16(order.Uint16(bs))
  134. case *uint16:
  135. *data = order.Uint16(bs)
  136. case *int32:
  137. *data = int32(order.Uint32(bs))
  138. case *uint32:
  139. *data = order.Uint32(bs)
  140. case *int64:
  141. *data = int64(order.Uint64(bs))
  142. case *uint64:
  143. *data = order.Uint64(bs)
  144. case []int8:
  145. for i, x := range bs { // Easier to loop over the input for 8-bit values.
  146. data[i] = int8(x)
  147. }
  148. case []uint8:
  149. copy(data, bs)
  150. case []int16:
  151. for i := range data {
  152. data[i] = int16(order.Uint16(bs[2*i:]))
  153. }
  154. case []uint16:
  155. for i := range data {
  156. data[i] = order.Uint16(bs[2*i:])
  157. }
  158. case []int32:
  159. for i := range data {
  160. data[i] = int32(order.Uint32(bs[4*i:]))
  161. }
  162. case []uint32:
  163. for i := range data {
  164. data[i] = order.Uint32(bs[4*i:])
  165. }
  166. case []int64:
  167. for i := range data {
  168. data[i] = int64(order.Uint64(bs[8*i:]))
  169. }
  170. case []uint64:
  171. for i := range data {
  172. data[i] = order.Uint64(bs[8*i:])
  173. }
  174. }
  175. return nil
  176. }
  177. // Fallback to reflect-based decoding.
  178. v := reflect.ValueOf(data)
  179. size := -1
  180. switch v.Kind() {
  181. case reflect.Ptr:
  182. v = v.Elem()
  183. size = dataSize(v)
  184. case reflect.Slice:
  185. size = dataSize(v)
  186. }
  187. if size < 0 {
  188. return errors.New("binary.Read: invalid type " + reflect.TypeOf(data).String())
  189. }
  190. d := &decoder{order: order, buf: make([]byte, size)}
  191. if _, err := io.ReadFull(r, d.buf); err != nil {
  192. return err
  193. }
  194. d.value(v)
  195. return nil
  196. }
  197. // Write writes the binary representation of data into w.
  198. // Data must be a fixed-size value or a slice of fixed-size
  199. // values, or a pointer to such data.
  200. // Bytes written to w are encoded using the specified byte order
  201. // and read from successive fields of the data.
  202. // When writing structs, zero values are written for fields
  203. // with blank (_) field names.
  204. func Write(w io.Writer, order ByteOrder, data interface{}) error {
  205. // Fast path for basic types and slices.
  206. if n := intDataSize(data); n != 0 {
  207. var b [8]byte
  208. var bs []byte
  209. if n > len(b) {
  210. bs = make([]byte, n)
  211. } else {
  212. bs = b[:n]
  213. }
  214. switch v := data.(type) {
  215. case *int8:
  216. bs = b[:1]
  217. b[0] = byte(*v)
  218. case int8:
  219. bs = b[:1]
  220. b[0] = byte(v)
  221. case []int8:
  222. for i, x := range v {
  223. bs[i] = byte(x)
  224. }
  225. case *uint8:
  226. bs = b[:1]
  227. b[0] = *v
  228. case uint8:
  229. bs = b[:1]
  230. b[0] = byte(v)
  231. case []uint8:
  232. bs = v
  233. case *int16:
  234. bs = b[:2]
  235. order.PutUint16(bs, uint16(*v))
  236. case int16:
  237. bs = b[:2]
  238. order.PutUint16(bs, uint16(v))
  239. case []int16:
  240. for i, x := range v {
  241. order.PutUint16(bs[2*i:], uint16(x))
  242. }
  243. case *uint16:
  244. bs = b[:2]
  245. order.PutUint16(bs, *v)
  246. case uint16:
  247. bs = b[:2]
  248. order.PutUint16(bs, v)
  249. case []uint16:
  250. for i, x := range v {
  251. order.PutUint16(bs[2*i:], x)
  252. }
  253. case *int32:
  254. bs = b[:4]
  255. order.PutUint32(bs, uint32(*v))
  256. case int32:
  257. bs = b[:4]
  258. order.PutUint32(bs, uint32(v))
  259. case []int32:
  260. for i, x := range v {
  261. order.PutUint32(bs[4*i:], uint32(x))
  262. }
  263. case *uint32:
  264. bs = b[:4]
  265. order.PutUint32(bs, *v)
  266. case uint32:
  267. bs = b[:4]
  268. order.PutUint32(bs, v)
  269. case []uint32:
  270. for i, x := range v {
  271. order.PutUint32(bs[4*i:], x)
  272. }
  273. case *int64:
  274. bs = b[:8]
  275. order.PutUint64(bs, uint64(*v))
  276. case int64:
  277. bs = b[:8]
  278. order.PutUint64(bs, uint64(v))
  279. case []int64:
  280. for i, x := range v {
  281. order.PutUint64(bs[8*i:], uint64(x))
  282. }
  283. case *uint64:
  284. bs = b[:8]
  285. order.PutUint64(bs, *v)
  286. case uint64:
  287. bs = b[:8]
  288. order.PutUint64(bs, v)
  289. case []uint64:
  290. for i, x := range v {
  291. order.PutUint64(bs[8*i:], x)
  292. }
  293. }
  294. _, err := w.Write(bs)
  295. return err
  296. }
  297. // Fallback to reflect-based encoding.
  298. v := reflect.Indirect(reflect.ValueOf(data))
  299. size := dataSize(v)
  300. if size < 0 {
  301. return errors.New("binary.Write: invalid type " + reflect.TypeOf(data).String())
  302. }
  303. buf := make([]byte, size)
  304. e := &encoder{order: order, buf: buf}
  305. e.value(v)
  306. _, err := w.Write(buf)
  307. return err
  308. }
  309. // Size returns how many bytes Write would generate to encode the value v, which
  310. // must be a fixed-size value or a slice of fixed-size values, or a pointer to such data.
  311. // If v is neither of these, Size returns -1.
  312. func Size(v interface{}) int {
  313. return dataSize(reflect.Indirect(reflect.ValueOf(v)))
  314. }
  315. // dataSize returns the number of bytes the actual data represented by v occupies in memory.
  316. // For compound structures, it sums the sizes of the elements. Thus, for instance, for a slice
  317. // it returns the length of the slice times the element size and does not count the memory
  318. // occupied by the header. If the type of v is not acceptable, dataSize returns -1.
  319. func dataSize(v reflect.Value) int {
  320. if v.Kind() == reflect.Slice {
  321. if s := sizeof(v.Type().Elem()); s >= 0 {
  322. return s * v.Len()
  323. }
  324. return -1
  325. }
  326. return sizeof(v.Type())
  327. }
  328. // sizeof returns the size >= 0 of variables for the given type or -1 if the type is not acceptable.
  329. func sizeof(t reflect.Type) int {
  330. switch t.Kind() {
  331. case reflect.Array:
  332. if s := sizeof(t.Elem()); s >= 0 {
  333. return s * t.Len()
  334. }
  335. case reflect.Struct:
  336. sum := 0
  337. for i, n := 0, t.NumField(); i < n; i++ {
  338. s := sizeof(t.Field(i).Type)
  339. if s < 0 {
  340. return -1
  341. }
  342. sum += s
  343. }
  344. return sum
  345. case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
  346. reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
  347. reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128, reflect.Ptr:
  348. return int(t.Size())
  349. }
  350. return -1
  351. }
  352. type coder struct {
  353. order ByteOrder
  354. buf []byte
  355. }
  356. type decoder coder
  357. type encoder coder
  358. func (d *decoder) uint8() uint8 {
  359. x := d.buf[0]
  360. d.buf = d.buf[1:]
  361. return x
  362. }
  363. func (e *encoder) uint8(x uint8) {
  364. e.buf[0] = x
  365. e.buf = e.buf[1:]
  366. }
  367. func (d *decoder) uint16() uint16 {
  368. x := d.order.Uint16(d.buf[0:2])
  369. d.buf = d.buf[2:]
  370. return x
  371. }
  372. func (e *encoder) uint16(x uint16) {
  373. e.order.PutUint16(e.buf[0:2], x)
  374. e.buf = e.buf[2:]
  375. }
  376. func (d *decoder) uint32() uint32 {
  377. x := d.order.Uint32(d.buf[0:4])
  378. d.buf = d.buf[4:]
  379. return x
  380. }
  381. func (e *encoder) uint32(x uint32) {
  382. e.order.PutUint32(e.buf[0:4], x)
  383. e.buf = e.buf[4:]
  384. }
  385. func (d *decoder) uint64() uint64 {
  386. x := d.order.Uint64(d.buf[0:8])
  387. d.buf = d.buf[8:]
  388. return x
  389. }
  390. func (e *encoder) uint64(x uint64) {
  391. e.order.PutUint64(e.buf[0:8], x)
  392. e.buf = e.buf[8:]
  393. }
  394. func (d *decoder) int8() int8 { return int8(d.uint8()) }
  395. func (e *encoder) int8(x int8) { e.uint8(uint8(x)) }
  396. func (d *decoder) int16() int16 { return int16(d.uint16()) }
  397. func (e *encoder) int16(x int16) { e.uint16(uint16(x)) }
  398. func (d *decoder) int32() int32 { return int32(d.uint32()) }
  399. func (e *encoder) int32(x int32) { e.uint32(uint32(x)) }
  400. func (d *decoder) int64() int64 { return int64(d.uint64()) }
  401. func (e *encoder) int64(x int64) { e.uint64(uint64(x)) }
  402. func (d *decoder) value(v reflect.Value) {
  403. switch v.Kind() {
  404. case reflect.Array:
  405. l := v.Len()
  406. for i := 0; i < l; i++ {
  407. d.value(v.Index(i))
  408. }
  409. case reflect.Struct:
  410. t := v.Type()
  411. l := v.NumField()
  412. for i := 0; i < l; i++ {
  413. // Note: Calling v.CanSet() below is an optimization.
  414. // It would be sufficient to check the field name,
  415. // but creating the StructField info for each field is
  416. // costly (run "go test -bench=ReadStruct" and compare
  417. // results when making changes to this code).
  418. if v := v.Field(i); v.CanSet() || t.Field(i).Name != "_" {
  419. d.value(v)
  420. } else {
  421. d.skip(v)
  422. }
  423. }
  424. case reflect.Slice:
  425. l := v.Len()
  426. for i := 0; i < l; i++ {
  427. d.value(v.Index(i))
  428. }
  429. case reflect.Int8:
  430. v.SetInt(int64(d.int8()))
  431. case reflect.Int16:
  432. v.SetInt(int64(d.int16()))
  433. case reflect.Int32:
  434. v.SetInt(int64(d.int32()))
  435. case reflect.Int64:
  436. v.SetInt(d.int64())
  437. case reflect.Uint8:
  438. v.SetUint(uint64(d.uint8()))
  439. case reflect.Uint16:
  440. v.SetUint(uint64(d.uint16()))
  441. case reflect.Uint32:
  442. v.SetUint(uint64(d.uint32()))
  443. case reflect.Uint64:
  444. v.SetUint(d.uint64())
  445. case reflect.Float32:
  446. v.SetFloat(float64(math.Float32frombits(d.uint32())))
  447. case reflect.Float64:
  448. v.SetFloat(math.Float64frombits(d.uint64()))
  449. case reflect.Complex64:
  450. v.SetComplex(complex(
  451. float64(math.Float32frombits(d.uint32())),
  452. float64(math.Float32frombits(d.uint32())),
  453. ))
  454. case reflect.Complex128:
  455. v.SetComplex(complex(
  456. math.Float64frombits(d.uint64()),
  457. math.Float64frombits(d.uint64()),
  458. ))
  459. }
  460. }
  461. func (e *encoder) value(v reflect.Value) {
  462. switch v.Kind() {
  463. case reflect.Array:
  464. l := v.Len()
  465. for i := 0; i < l; i++ {
  466. e.value(v.Index(i))
  467. }
  468. case reflect.Struct:
  469. t := v.Type()
  470. l := v.NumField()
  471. for i := 0; i < l; i++ {
  472. // see comment for corresponding code in decoder.value()
  473. if v := v.Field(i); v.CanSet() || t.Field(i).Name != "_" {
  474. e.value(v)
  475. } else {
  476. e.skip(v)
  477. }
  478. }
  479. case reflect.Slice:
  480. l := v.Len()
  481. for i := 0; i < l; i++ {
  482. e.value(v.Index(i))
  483. }
  484. case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
  485. switch v.Type().Kind() {
  486. case reflect.Int8:
  487. e.int8(int8(v.Int()))
  488. case reflect.Int16:
  489. e.int16(int16(v.Int()))
  490. case reflect.Int32:
  491. e.int32(int32(v.Int()))
  492. case reflect.Int64:
  493. e.int64(v.Int())
  494. }
  495. case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
  496. switch v.Type().Kind() {
  497. case reflect.Uint8:
  498. e.uint8(uint8(v.Uint()))
  499. case reflect.Uint16:
  500. e.uint16(uint16(v.Uint()))
  501. case reflect.Uint32:
  502. e.uint32(uint32(v.Uint()))
  503. case reflect.Uint64:
  504. e.uint64(v.Uint())
  505. }
  506. case reflect.Float32, reflect.Float64:
  507. switch v.Type().Kind() {
  508. case reflect.Float32:
  509. e.uint32(math.Float32bits(float32(v.Float())))
  510. case reflect.Float64:
  511. e.uint64(math.Float64bits(v.Float()))
  512. }
  513. case reflect.Complex64, reflect.Complex128:
  514. switch v.Type().Kind() {
  515. case reflect.Complex64:
  516. x := v.Complex()
  517. e.uint32(math.Float32bits(float32(real(x))))
  518. e.uint32(math.Float32bits(float32(imag(x))))
  519. case reflect.Complex128:
  520. x := v.Complex()
  521. e.uint64(math.Float64bits(real(x)))
  522. e.uint64(math.Float64bits(imag(x)))
  523. }
  524. }
  525. }
  526. func (d *decoder) skip(v reflect.Value) {
  527. d.buf = d.buf[dataSize(v):]
  528. }
  529. func (e *encoder) skip(v reflect.Value) {
  530. n := dataSize(v)
  531. for i := range e.buf[0:n] {
  532. e.buf[i] = 0
  533. }
  534. e.buf = e.buf[n:]
  535. }
  536. // intDataSize returns the size of the data required to represent the data when encoded.
  537. // It returns zero if the type cannot be implemented by the fast path in Read or Write.
  538. func intDataSize(data interface{}) int {
  539. switch data := data.(type) {
  540. case int8, *int8, *uint8:
  541. return 1
  542. case []int8:
  543. return len(data)
  544. case []uint8:
  545. return len(data)
  546. case int16, *int16, *uint16:
  547. return 2
  548. case []int16:
  549. return 2 * len(data)
  550. case []uint16:
  551. return 2 * len(data)
  552. case int32, *int32, *uint32:
  553. return 4
  554. case []int32:
  555. return 4 * len(data)
  556. case []uint32:
  557. return 4 * len(data)
  558. case int64, *int64, *uint64:
  559. return 8
  560. case []int64:
  561. return 8 * len(data)
  562. case []uint64:
  563. return 8 * len(data)
  564. }
  565. return 0
  566. }