decode.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685
  1. package yaml
  2. import (
  3. "encoding"
  4. "encoding/base64"
  5. "fmt"
  6. "math"
  7. "reflect"
  8. "strconv"
  9. "time"
  10. )
  11. const (
  12. documentNode = 1 << iota
  13. mappingNode
  14. sequenceNode
  15. scalarNode
  16. aliasNode
  17. )
  18. type node struct {
  19. kind int
  20. line, column int
  21. tag string
  22. value string
  23. implicit bool
  24. children []*node
  25. anchors map[string]*node
  26. }
  27. // ----------------------------------------------------------------------------
  28. // Parser, produces a node tree out of a libyaml event stream.
  29. type parser struct {
  30. parser yaml_parser_t
  31. event yaml_event_t
  32. doc *node
  33. }
  34. func newParser(b []byte) *parser {
  35. p := parser{}
  36. if !yaml_parser_initialize(&p.parser) {
  37. panic("failed to initialize YAML emitter")
  38. }
  39. if len(b) == 0 {
  40. b = []byte{'\n'}
  41. }
  42. yaml_parser_set_input_string(&p.parser, b)
  43. p.skip()
  44. if p.event.typ != yaml_STREAM_START_EVENT {
  45. panic("expected stream start event, got " + strconv.Itoa(int(p.event.typ)))
  46. }
  47. p.skip()
  48. return &p
  49. }
  50. func (p *parser) destroy() {
  51. if p.event.typ != yaml_NO_EVENT {
  52. yaml_event_delete(&p.event)
  53. }
  54. yaml_parser_delete(&p.parser)
  55. }
  56. func (p *parser) skip() {
  57. if p.event.typ != yaml_NO_EVENT {
  58. if p.event.typ == yaml_STREAM_END_EVENT {
  59. failf("attempted to go past the end of stream; corrupted value?")
  60. }
  61. yaml_event_delete(&p.event)
  62. }
  63. if !yaml_parser_parse(&p.parser, &p.event) {
  64. p.fail()
  65. }
  66. }
  67. func (p *parser) fail() {
  68. var where string
  69. var line int
  70. if p.parser.problem_mark.line != 0 {
  71. line = p.parser.problem_mark.line
  72. } else if p.parser.context_mark.line != 0 {
  73. line = p.parser.context_mark.line
  74. }
  75. if line != 0 {
  76. where = "line " + strconv.Itoa(line) + ": "
  77. }
  78. var msg string
  79. if len(p.parser.problem) > 0 {
  80. msg = p.parser.problem
  81. } else {
  82. msg = "unknown problem parsing YAML content"
  83. }
  84. failf("%s%s", where, msg)
  85. }
  86. func (p *parser) anchor(n *node, anchor []byte) {
  87. if anchor != nil {
  88. p.doc.anchors[string(anchor)] = n
  89. }
  90. }
  91. func (p *parser) parse() *node {
  92. switch p.event.typ {
  93. case yaml_SCALAR_EVENT:
  94. return p.scalar()
  95. case yaml_ALIAS_EVENT:
  96. return p.alias()
  97. case yaml_MAPPING_START_EVENT:
  98. return p.mapping()
  99. case yaml_SEQUENCE_START_EVENT:
  100. return p.sequence()
  101. case yaml_DOCUMENT_START_EVENT:
  102. return p.document()
  103. case yaml_STREAM_END_EVENT:
  104. // Happens when attempting to decode an empty buffer.
  105. return nil
  106. default:
  107. panic("attempted to parse unknown event: " + strconv.Itoa(int(p.event.typ)))
  108. }
  109. }
  110. func (p *parser) node(kind int) *node {
  111. return &node{
  112. kind: kind,
  113. line: p.event.start_mark.line,
  114. column: p.event.start_mark.column,
  115. }
  116. }
  117. func (p *parser) document() *node {
  118. n := p.node(documentNode)
  119. n.anchors = make(map[string]*node)
  120. p.doc = n
  121. p.skip()
  122. n.children = append(n.children, p.parse())
  123. if p.event.typ != yaml_DOCUMENT_END_EVENT {
  124. panic("expected end of document event but got " + strconv.Itoa(int(p.event.typ)))
  125. }
  126. p.skip()
  127. return n
  128. }
  129. func (p *parser) alias() *node {
  130. n := p.node(aliasNode)
  131. n.value = string(p.event.anchor)
  132. p.skip()
  133. return n
  134. }
  135. func (p *parser) scalar() *node {
  136. n := p.node(scalarNode)
  137. n.value = string(p.event.value)
  138. n.tag = string(p.event.tag)
  139. n.implicit = p.event.implicit
  140. p.anchor(n, p.event.anchor)
  141. p.skip()
  142. return n
  143. }
  144. func (p *parser) sequence() *node {
  145. n := p.node(sequenceNode)
  146. p.anchor(n, p.event.anchor)
  147. p.skip()
  148. for p.event.typ != yaml_SEQUENCE_END_EVENT {
  149. n.children = append(n.children, p.parse())
  150. }
  151. p.skip()
  152. return n
  153. }
  154. func (p *parser) mapping() *node {
  155. n := p.node(mappingNode)
  156. p.anchor(n, p.event.anchor)
  157. p.skip()
  158. for p.event.typ != yaml_MAPPING_END_EVENT {
  159. n.children = append(n.children, p.parse(), p.parse())
  160. }
  161. p.skip()
  162. return n
  163. }
  164. // ----------------------------------------------------------------------------
  165. // Decoder, unmarshals a node into a provided value.
  166. type decoder struct {
  167. doc *node
  168. aliases map[string]bool
  169. mapType reflect.Type
  170. terrors []string
  171. strict bool
  172. }
  173. var (
  174. mapItemType = reflect.TypeOf(MapItem{})
  175. durationType = reflect.TypeOf(time.Duration(0))
  176. defaultMapType = reflect.TypeOf(map[interface{}]interface{}{})
  177. ifaceType = defaultMapType.Elem()
  178. )
  179. func newDecoder(strict bool) *decoder {
  180. d := &decoder{mapType: defaultMapType, strict: strict}
  181. d.aliases = make(map[string]bool)
  182. return d
  183. }
  184. func (d *decoder) terror(n *node, tag string, out reflect.Value) {
  185. if n.tag != "" {
  186. tag = n.tag
  187. }
  188. value := n.value
  189. if tag != yaml_SEQ_TAG && tag != yaml_MAP_TAG {
  190. if len(value) > 10 {
  191. value = " `" + value[:7] + "...`"
  192. } else {
  193. value = " `" + value + "`"
  194. }
  195. }
  196. d.terrors = append(d.terrors, fmt.Sprintf("line %d: cannot unmarshal %s%s into %s", n.line+1, shortTag(tag), value, out.Type()))
  197. }
  198. func (d *decoder) callUnmarshaler(n *node, u Unmarshaler) (good bool) {
  199. terrlen := len(d.terrors)
  200. err := u.UnmarshalYAML(func(v interface{}) (err error) {
  201. defer handleErr(&err)
  202. d.unmarshal(n, reflect.ValueOf(v))
  203. if len(d.terrors) > terrlen {
  204. issues := d.terrors[terrlen:]
  205. d.terrors = d.terrors[:terrlen]
  206. return &TypeError{issues}
  207. }
  208. return nil
  209. })
  210. if e, ok := err.(*TypeError); ok {
  211. d.terrors = append(d.terrors, e.Errors...)
  212. return false
  213. }
  214. if err != nil {
  215. fail(err)
  216. }
  217. return true
  218. }
  219. // d.prepare initializes and dereferences pointers and calls UnmarshalYAML
  220. // if a value is found to implement it.
  221. // It returns the initialized and dereferenced out value, whether
  222. // unmarshalling was already done by UnmarshalYAML, and if so whether
  223. // its types unmarshalled appropriately.
  224. //
  225. // If n holds a null value, prepare returns before doing anything.
  226. func (d *decoder) prepare(n *node, out reflect.Value) (newout reflect.Value, unmarshaled, good bool) {
  227. if n.tag == yaml_NULL_TAG || n.kind == scalarNode && n.tag == "" && (n.value == "null" || n.value == "" && n.implicit) {
  228. return out, false, false
  229. }
  230. again := true
  231. for again {
  232. again = false
  233. if out.Kind() == reflect.Ptr {
  234. if out.IsNil() {
  235. out.Set(reflect.New(out.Type().Elem()))
  236. }
  237. out = out.Elem()
  238. again = true
  239. }
  240. if out.CanAddr() {
  241. if u, ok := out.Addr().Interface().(Unmarshaler); ok {
  242. good = d.callUnmarshaler(n, u)
  243. return out, true, good
  244. }
  245. }
  246. }
  247. return out, false, false
  248. }
  249. func (d *decoder) unmarshal(n *node, out reflect.Value) (good bool) {
  250. switch n.kind {
  251. case documentNode:
  252. return d.document(n, out)
  253. case aliasNode:
  254. return d.alias(n, out)
  255. }
  256. out, unmarshaled, good := d.prepare(n, out)
  257. if unmarshaled {
  258. return good
  259. }
  260. switch n.kind {
  261. case scalarNode:
  262. good = d.scalar(n, out)
  263. case mappingNode:
  264. good = d.mapping(n, out)
  265. case sequenceNode:
  266. good = d.sequence(n, out)
  267. default:
  268. panic("internal error: unknown node kind: " + strconv.Itoa(n.kind))
  269. }
  270. return good
  271. }
  272. func (d *decoder) document(n *node, out reflect.Value) (good bool) {
  273. if len(n.children) == 1 {
  274. d.doc = n
  275. d.unmarshal(n.children[0], out)
  276. return true
  277. }
  278. return false
  279. }
  280. func (d *decoder) alias(n *node, out reflect.Value) (good bool) {
  281. an, ok := d.doc.anchors[n.value]
  282. if !ok {
  283. failf("unknown anchor '%s' referenced", n.value)
  284. }
  285. if d.aliases[n.value] {
  286. failf("anchor '%s' value contains itself", n.value)
  287. }
  288. d.aliases[n.value] = true
  289. good = d.unmarshal(an, out)
  290. delete(d.aliases, n.value)
  291. return good
  292. }
  293. var zeroValue reflect.Value
  294. func resetMap(out reflect.Value) {
  295. for _, k := range out.MapKeys() {
  296. out.SetMapIndex(k, zeroValue)
  297. }
  298. }
  299. func (d *decoder) scalar(n *node, out reflect.Value) (good bool) {
  300. var tag string
  301. var resolved interface{}
  302. if n.tag == "" && !n.implicit {
  303. tag = yaml_STR_TAG
  304. resolved = n.value
  305. } else {
  306. tag, resolved = resolve(n.tag, n.value)
  307. if tag == yaml_BINARY_TAG {
  308. data, err := base64.StdEncoding.DecodeString(resolved.(string))
  309. if err != nil {
  310. failf("!!binary value contains invalid base64 data")
  311. }
  312. resolved = string(data)
  313. }
  314. }
  315. if resolved == nil {
  316. if out.Kind() == reflect.Map && !out.CanAddr() {
  317. resetMap(out)
  318. } else {
  319. out.Set(reflect.Zero(out.Type()))
  320. }
  321. return true
  322. }
  323. if s, ok := resolved.(string); ok && out.CanAddr() {
  324. if u, ok := out.Addr().Interface().(encoding.TextUnmarshaler); ok {
  325. err := u.UnmarshalText([]byte(s))
  326. if err != nil {
  327. fail(err)
  328. }
  329. return true
  330. }
  331. }
  332. switch out.Kind() {
  333. case reflect.String:
  334. if tag == yaml_BINARY_TAG {
  335. out.SetString(resolved.(string))
  336. good = true
  337. } else if resolved != nil {
  338. out.SetString(n.value)
  339. good = true
  340. }
  341. case reflect.Interface:
  342. if resolved == nil {
  343. out.Set(reflect.Zero(out.Type()))
  344. } else {
  345. out.Set(reflect.ValueOf(resolved))
  346. }
  347. good = true
  348. case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
  349. switch resolved := resolved.(type) {
  350. case int:
  351. if !out.OverflowInt(int64(resolved)) {
  352. out.SetInt(int64(resolved))
  353. good = true
  354. }
  355. case int64:
  356. if !out.OverflowInt(resolved) {
  357. out.SetInt(resolved)
  358. good = true
  359. }
  360. case uint64:
  361. if resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) {
  362. out.SetInt(int64(resolved))
  363. good = true
  364. }
  365. case float64:
  366. if resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) {
  367. out.SetInt(int64(resolved))
  368. good = true
  369. }
  370. case string:
  371. if out.Type() == durationType {
  372. d, err := time.ParseDuration(resolved)
  373. if err == nil {
  374. out.SetInt(int64(d))
  375. good = true
  376. }
  377. }
  378. }
  379. case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
  380. switch resolved := resolved.(type) {
  381. case int:
  382. if resolved >= 0 && !out.OverflowUint(uint64(resolved)) {
  383. out.SetUint(uint64(resolved))
  384. good = true
  385. }
  386. case int64:
  387. if resolved >= 0 && !out.OverflowUint(uint64(resolved)) {
  388. out.SetUint(uint64(resolved))
  389. good = true
  390. }
  391. case uint64:
  392. if !out.OverflowUint(uint64(resolved)) {
  393. out.SetUint(uint64(resolved))
  394. good = true
  395. }
  396. case float64:
  397. if resolved <= math.MaxUint64 && !out.OverflowUint(uint64(resolved)) {
  398. out.SetUint(uint64(resolved))
  399. good = true
  400. }
  401. }
  402. case reflect.Bool:
  403. switch resolved := resolved.(type) {
  404. case bool:
  405. out.SetBool(resolved)
  406. good = true
  407. }
  408. case reflect.Float32, reflect.Float64:
  409. switch resolved := resolved.(type) {
  410. case int:
  411. out.SetFloat(float64(resolved))
  412. good = true
  413. case int64:
  414. out.SetFloat(float64(resolved))
  415. good = true
  416. case uint64:
  417. out.SetFloat(float64(resolved))
  418. good = true
  419. case float64:
  420. out.SetFloat(resolved)
  421. good = true
  422. }
  423. case reflect.Ptr:
  424. if out.Type().Elem() == reflect.TypeOf(resolved) {
  425. // TODO DOes this make sense? When is out a Ptr except when decoding a nil value?
  426. elem := reflect.New(out.Type().Elem())
  427. elem.Elem().Set(reflect.ValueOf(resolved))
  428. out.Set(elem)
  429. good = true
  430. }
  431. }
  432. if !good {
  433. d.terror(n, tag, out)
  434. }
  435. return good
  436. }
  437. func settableValueOf(i interface{}) reflect.Value {
  438. v := reflect.ValueOf(i)
  439. sv := reflect.New(v.Type()).Elem()
  440. sv.Set(v)
  441. return sv
  442. }
  443. func (d *decoder) sequence(n *node, out reflect.Value) (good bool) {
  444. l := len(n.children)
  445. var iface reflect.Value
  446. switch out.Kind() {
  447. case reflect.Slice:
  448. out.Set(reflect.MakeSlice(out.Type(), l, l))
  449. case reflect.Interface:
  450. // No type hints. Will have to use a generic sequence.
  451. iface = out
  452. out = settableValueOf(make([]interface{}, l))
  453. default:
  454. d.terror(n, yaml_SEQ_TAG, out)
  455. return false
  456. }
  457. et := out.Type().Elem()
  458. j := 0
  459. for i := 0; i < l; i++ {
  460. e := reflect.New(et).Elem()
  461. if ok := d.unmarshal(n.children[i], e); ok {
  462. out.Index(j).Set(e)
  463. j++
  464. }
  465. }
  466. out.Set(out.Slice(0, j))
  467. if iface.IsValid() {
  468. iface.Set(out)
  469. }
  470. return true
  471. }
  472. func (d *decoder) mapping(n *node, out reflect.Value) (good bool) {
  473. switch out.Kind() {
  474. case reflect.Struct:
  475. return d.mappingStruct(n, out)
  476. case reflect.Slice:
  477. return d.mappingSlice(n, out)
  478. case reflect.Map:
  479. // okay
  480. case reflect.Interface:
  481. if d.mapType.Kind() == reflect.Map {
  482. iface := out
  483. out = reflect.MakeMap(d.mapType)
  484. iface.Set(out)
  485. } else {
  486. slicev := reflect.New(d.mapType).Elem()
  487. if !d.mappingSlice(n, slicev) {
  488. return false
  489. }
  490. out.Set(slicev)
  491. return true
  492. }
  493. default:
  494. d.terror(n, yaml_MAP_TAG, out)
  495. return false
  496. }
  497. outt := out.Type()
  498. kt := outt.Key()
  499. et := outt.Elem()
  500. mapType := d.mapType
  501. if outt.Key() == ifaceType && outt.Elem() == ifaceType {
  502. d.mapType = outt
  503. }
  504. if out.IsNil() {
  505. out.Set(reflect.MakeMap(outt))
  506. }
  507. l := len(n.children)
  508. for i := 0; i < l; i += 2 {
  509. if isMerge(n.children[i]) {
  510. d.merge(n.children[i+1], out)
  511. continue
  512. }
  513. k := reflect.New(kt).Elem()
  514. if d.unmarshal(n.children[i], k) {
  515. kkind := k.Kind()
  516. if kkind == reflect.Interface {
  517. kkind = k.Elem().Kind()
  518. }
  519. if kkind == reflect.Map || kkind == reflect.Slice {
  520. failf("invalid map key: %#v", k.Interface())
  521. }
  522. e := reflect.New(et).Elem()
  523. if d.unmarshal(n.children[i+1], e) {
  524. out.SetMapIndex(k, e)
  525. }
  526. }
  527. }
  528. d.mapType = mapType
  529. return true
  530. }
  531. func (d *decoder) mappingSlice(n *node, out reflect.Value) (good bool) {
  532. outt := out.Type()
  533. if outt.Elem() != mapItemType {
  534. d.terror(n, yaml_MAP_TAG, out)
  535. return false
  536. }
  537. mapType := d.mapType
  538. d.mapType = outt
  539. var slice []MapItem
  540. var l = len(n.children)
  541. for i := 0; i < l; i += 2 {
  542. if isMerge(n.children[i]) {
  543. d.merge(n.children[i+1], out)
  544. continue
  545. }
  546. item := MapItem{}
  547. k := reflect.ValueOf(&item.Key).Elem()
  548. if d.unmarshal(n.children[i], k) {
  549. v := reflect.ValueOf(&item.Value).Elem()
  550. if d.unmarshal(n.children[i+1], v) {
  551. slice = append(slice, item)
  552. }
  553. }
  554. }
  555. out.Set(reflect.ValueOf(slice))
  556. d.mapType = mapType
  557. return true
  558. }
  559. func (d *decoder) mappingStruct(n *node, out reflect.Value) (good bool) {
  560. sinfo, err := getStructInfo(out.Type())
  561. if err != nil {
  562. panic(err)
  563. }
  564. name := settableValueOf("")
  565. l := len(n.children)
  566. var inlineMap reflect.Value
  567. var elemType reflect.Type
  568. if sinfo.InlineMap != -1 {
  569. inlineMap = out.Field(sinfo.InlineMap)
  570. inlineMap.Set(reflect.New(inlineMap.Type()).Elem())
  571. elemType = inlineMap.Type().Elem()
  572. }
  573. for i := 0; i < l; i += 2 {
  574. ni := n.children[i]
  575. if isMerge(ni) {
  576. d.merge(n.children[i+1], out)
  577. continue
  578. }
  579. if !d.unmarshal(ni, name) {
  580. continue
  581. }
  582. if info, ok := sinfo.FieldsMap[name.String()]; ok {
  583. var field reflect.Value
  584. if info.Inline == nil {
  585. field = out.Field(info.Num)
  586. } else {
  587. field = out.FieldByIndex(info.Inline)
  588. }
  589. d.unmarshal(n.children[i+1], field)
  590. } else if sinfo.InlineMap != -1 {
  591. if inlineMap.IsNil() {
  592. inlineMap.Set(reflect.MakeMap(inlineMap.Type()))
  593. }
  594. value := reflect.New(elemType).Elem()
  595. d.unmarshal(n.children[i+1], value)
  596. inlineMap.SetMapIndex(name, value)
  597. } else if d.strict {
  598. d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s not found in struct %s", n.line+1, name.String(), out.Type()))
  599. }
  600. }
  601. return true
  602. }
  603. func failWantMap() {
  604. failf("map merge requires map or sequence of maps as the value")
  605. }
  606. func (d *decoder) merge(n *node, out reflect.Value) {
  607. switch n.kind {
  608. case mappingNode:
  609. d.unmarshal(n, out)
  610. case aliasNode:
  611. an, ok := d.doc.anchors[n.value]
  612. if ok && an.kind != mappingNode {
  613. failWantMap()
  614. }
  615. d.unmarshal(n, out)
  616. case sequenceNode:
  617. // Step backwards as earlier nodes take precedence.
  618. for i := len(n.children) - 1; i >= 0; i-- {
  619. ni := n.children[i]
  620. if ni.kind == aliasNode {
  621. an, ok := d.doc.anchors[ni.value]
  622. if ok && an.kind != mappingNode {
  623. failWantMap()
  624. }
  625. } else if ni.kind != mappingNode {
  626. failWantMap()
  627. }
  628. d.unmarshal(ni, out)
  629. }
  630. default:
  631. failWantMap()
  632. }
  633. }
  634. func isMerge(n *node) bool {
  635. return n.kind == scalarNode && n.value == "<<" && (n.implicit == true || n.tag == yaml_MERGE_TAG)
  636. }