package mxj
import (
"bytes"
"encoding/xml"
"reflect"
)
const (
DefaultElementTag = "element"
)
// Encode arbitrary value as XML.
//
// Note: unmarshaling the resultant
// XML may not return the original value, since tag labels may have been injected
// to create the XML representation of the value.
/*
Encode an arbitrary JSON object.
package main
import (
"encoding/json"
"fmt"
"github.com/clbanning/mxj/v2"
)
func main() {
jsondata := []byte(`[
{ "somekey":"somevalue" },
"string",
3.14159265,
true
]`)
var i interface{}
err := json.Unmarshal(jsondata, &i)
if err != nil {
// do something
}
x, err := mxj.AnyXmlIndent(i, "", " ", "mydoc")
if err != nil {
// do something else
}
fmt.Println(string(x))
}
output:
<mydoc>
<somekey>somevalue</somekey>
<element>string</element>
<element>3.14159265</element>
<element>true</element>
</mydoc>
An extreme example is available in examples/goofy_map.go.
*/
// Alternative values for DefaultRootTag and DefaultElementTag can be set as:
// AnyXml( v, myRootTag, myElementTag).
func AnyXml(v interface{}, tags ...string) ([]byte, error) {
var rt, et string
if len(tags) == 1 || len(tags) == 2 {
rt = tags[0]
} else {
rt = DefaultRootTag
}
if len(tags) == 2 {
et = tags[1]
} else {
et = DefaultElementTag
}
if v == nil {
if useGoXmlEmptyElemSyntax {
return []byte("<" + rt + "></" + rt + ">"), nil
}
return []byte("<" + rt + "/>"), nil
}
if reflect.TypeOf(v).Kind() == reflect.Struct {
return xml.Marshal(v)
}
var err error
s := new(bytes.Buffer)
p := new(pretty)
var b []byte
switch v.(type) {
case []interface{}:
if _, err = s.WriteString("<" + rt + ">"); err != nil {
return nil, err
}
for _, vv := range v.([]interface{}) {
switch vv.(type) {
case map[string]interface{}:
m := vv.(map[string]interface{})
if len(m) == 1 {
for tag, val := range m {
err = marshalMapToXmlIndent(false, s, tag, val, p)
}
} else {
err = marshalMapToXmlIndent(false, s, et, vv, p)
}
default:
err = marshalMapToXmlIndent(false, s, et, vv, p)
}
if err != nil {
break
}
}
if _, err = s.WriteString("</" + rt + ">"); err != nil {
return nil, err
}
b = s.Bytes()
case map[string]interface{}:
m := Map(v.(map[string]interface{}))
b, err = m.Xml(rt)
default:
err = marshalMapToXmlIndent(false, s, rt, v, p)
b = s.Bytes()
}
return b, err
}
// Encode an arbitrary value as a pretty XML string.
// Alternative values for DefaultRootTag and DefaultElementTag can be set as:
// AnyXmlIndent( v, "", " ", myRootTag, myElementTag).
func AnyXmlIndent(v interface{}, prefix, indent string, tags ...string) ([]byte, error) {
var rt, et string
if len(tags) == 1 || len(tags) == 2 {
rt = tags[0]
} else {
rt = DefaultRootTag
}
if len(tags) == 2 {
et = tags[1]
} else {
et = DefaultElementTag
}
if v == nil {
if useGoXmlEmptyElemSyntax {
return []byte(prefix + "<" + rt + "></" + rt + ">"), nil
}
return []byte(prefix + "<" + rt + "/>"), nil
}
if reflect.TypeOf(v).Kind() == reflect.Struct {
return xml.MarshalIndent(v, prefix, indent)
}
var err error
s := new(bytes.Buffer)
p := new(pretty)
p.indent = indent
p.padding = prefix
var b []byte
switch v.(type) {
case []interface{}:
if _, err = s.WriteString("<" + rt + ">\n"); err != nil {
return nil, err
}
p.Indent()
for _, vv := range v.([]interface{}) {
switch vv.(type) {
case map[string]interface{}:
m := vv.(map[string]interface{})
if len(m) == 1 {
for tag, val := range m {
err = marshalMapToXmlIndent(true, s, tag, val, p)
}
} else {
p.start = 1 // we 1 tag in
err = marshalMapToXmlIndent(true, s, et, vv, p)
// *s += "\n"
if _, err = s.WriteString("\n"); err != nil {
return nil, err
}
}
default:
p.start = 0 // in case trailing p.start = 1
err = marshalMapToXmlIndent(true, s, et, vv, p)
}
if err != nil {
break
}
}
if _, err = s.WriteString(`</` + rt + `>`); err != nil {
return nil, err
}
b = s.Bytes()
case map[string]interface{}:
m := Map(v.(map[string]interface{}))
b, err = m.XmlIndent(prefix, indent, rt)
default:
err = marshalMapToXmlIndent(true, s, rt, v, p)
b = s.Bytes()
}
return b, err
}
// Copyright 2016 Charles Banning. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file
package mxj
import (
"bytes"
)
var xmlEscapeChars bool
// XMLEscapeChars(true) forces escaping invalid characters in attribute and element values.
// NOTE: this is brute force with NO interrogation of '&' being escaped already; if it is
// then '&' will be re-escaped as '&amp;'.
//
/*
The values are:
" "
' '
< <
> >
& &
*/
//
// Note: if XMLEscapeCharsDecoder(true) has been called - or the default, 'false,' value
// has been toggled to 'true' - then XMLEscapeChars(true) is ignored. If XMLEscapeChars(true)
// has already been called before XMLEscapeCharsDecoder(true), XMLEscapeChars(false) is called
// to turn escape encoding on mv.Xml, etc., to prevent double escaping ampersands, '&'.
func XMLEscapeChars(b ...bool) {
var bb bool
if len(b) == 0 {
bb = !xmlEscapeChars
} else {
bb = b[0]
}
if bb == true && xmlEscapeCharsDecoder == false {
xmlEscapeChars = true
} else {
xmlEscapeChars = false
}
}
// Scan for '&' first, since 's' may contain "&" that is parsed to "&amp;"
// - or "<" that is parsed to "&lt;".
var escapechars = [][2][]byte{
{[]byte(`&`), []byte(`&`)},
{[]byte(`<`), []byte(`<`)},
{[]byte(`>`), []byte(`>`)},
{[]byte(`"`), []byte(`"`)},
{[]byte(`'`), []byte(`'`)},
}
func escapeChars(s string) string {
if len(s) == 0 {
return s
}
b := []byte(s)
for _, v := range escapechars {
n := bytes.Count(b, v[0])
if n == 0 {
continue
}
b = bytes.Replace(b, v[0], v[1], n)
}
return string(b)
}
// per issue #84, escape CharData values from xml.Decoder
var xmlEscapeCharsDecoder bool
// XMLEscapeCharsDecoder(b ...bool) escapes XML characters in xml.CharData values
// returned by Decoder.Token. Thus, the internal Map values will contain escaped
// values, and you do not need to set XMLEscapeChars for proper encoding.
//
// By default, the Map values have the non-escaped values returned by Decoder.Token.
// XMLEscapeCharsDecoder(true) - or, XMLEscapeCharsDecoder() - will toggle escape
// encoding 'on.'
//
// Note: if XMLEscapeCharDecoder(true) is call then XMLEscapeChars(false) is
// called to prevent re-escaping the values on encoding using mv.Xml, etc.
func XMLEscapeCharsDecoder(b ...bool) {
if len(b) == 0 {
xmlEscapeCharsDecoder = !xmlEscapeCharsDecoder
} else {
xmlEscapeCharsDecoder = b[0]
}
if xmlEscapeCharsDecoder == true && xmlEscapeChars == true {
xmlEscapeChars = false
}
}
package mxj
// Checks whether the path exists. If err != nil then 'false' is returned
// along with the error encountered parsing either the "path" or "subkeys"
// argument.
func (mv Map) Exists(path string, subkeys ...string) (bool, error) {
v, err := mv.ValuesForPath(path, subkeys...)
return (err == nil && len(v) > 0), err
}
package mxj
import (
"fmt"
"io"
"os"
)
type Maps []Map
func NewMaps() Maps {
return make(Maps, 0)
}
type MapRaw struct {
M Map
R []byte
}
// NewMapsFromXmlFile - creates an array from a file of JSON values.
func NewMapsFromJsonFile(name string) (Maps, error) {
fi, err := os.Stat(name)
if err != nil {
return nil, err
}
if !fi.Mode().IsRegular() {
return nil, fmt.Errorf("file %s is not a regular file", name)
}
fh, err := os.Open(name)
if err != nil {
return nil, err
}
defer fh.Close()
am := make([]Map, 0)
for {
m, raw, err := NewMapJsonReaderRaw(fh)
if err != nil && err != io.EOF {
return am, fmt.Errorf("error: %s - reading: %s", err.Error(), string(raw))
}
if len(m) > 0 {
am = append(am, m)
}
if err == io.EOF {
break
}
}
return am, nil
}
// ReadMapsFromJsonFileRaw - creates an array of MapRaw from a file of JSON values.
func NewMapsFromJsonFileRaw(name string) ([]MapRaw, error) {
fi, err := os.Stat(name)
if err != nil {
return nil, err
}
if !fi.Mode().IsRegular() {
return nil, fmt.Errorf("file %s is not a regular file", name)
}
fh, err := os.Open(name)
if err != nil {
return nil, err
}
defer fh.Close()
am := make([]MapRaw, 0)
for {
mr := new(MapRaw)
mr.M, mr.R, err = NewMapJsonReaderRaw(fh)
if err != nil && err != io.EOF {
return am, fmt.Errorf("error: %s - reading: %s", err.Error(), string(mr.R))
}
if len(mr.M) > 0 {
am = append(am, *mr)
}
if err == io.EOF {
break
}
}
return am, nil
}
// NewMapsFromXmlFile - creates an array from a file of XML values.
func NewMapsFromXmlFile(name string) (Maps, error) {
fi, err := os.Stat(name)
if err != nil {
return nil, err
}
if !fi.Mode().IsRegular() {
return nil, fmt.Errorf("file %s is not a regular file", name)
}
fh, err := os.Open(name)
if err != nil {
return nil, err
}
defer fh.Close()
am := make([]Map, 0)
for {
m, raw, err := NewMapXmlReaderRaw(fh)
if err != nil && err != io.EOF {
return am, fmt.Errorf("error: %s - reading: %s", err.Error(), string(raw))
}
if len(m) > 0 {
am = append(am, m)
}
if err == io.EOF {
break
}
}
return am, nil
}
// NewMapsFromXmlFileRaw - creates an array of MapRaw from a file of XML values.
// NOTE: the slice with the raw XML is clean with no extra capacity - unlike NewMapXmlReaderRaw().
// It is slow at parsing a file from disk and is intended for relatively small utility files.
func NewMapsFromXmlFileRaw(name string) ([]MapRaw, error) {
fi, err := os.Stat(name)
if err != nil {
return nil, err
}
if !fi.Mode().IsRegular() {
return nil, fmt.Errorf("file %s is not a regular file", name)
}
fh, err := os.Open(name)
if err != nil {
return nil, err
}
defer fh.Close()
am := make([]MapRaw, 0)
for {
mr := new(MapRaw)
mr.M, mr.R, err = NewMapXmlReaderRaw(fh)
if err != nil && err != io.EOF {
return am, fmt.Errorf("error: %s - reading: %s", err.Error(), string(mr.R))
}
if len(mr.M) > 0 {
am = append(am, *mr)
}
if err == io.EOF {
break
}
}
return am, nil
}
// ------------------------ Maps writing -------------------------
// These are handy-dandy methods for dumping configuration data, etc.
// JsonString - analogous to mv.Json()
func (mvs Maps) JsonString(safeEncoding ...bool) (string, error) {
var s string
for _, v := range mvs {
j, err := v.Json()
if err != nil {
return s, err
}
s += string(j)
}
return s, nil
}
// JsonStringIndent - analogous to mv.JsonIndent()
func (mvs Maps) JsonStringIndent(prefix, indent string, safeEncoding ...bool) (string, error) {
var s string
var haveFirst bool
for _, v := range mvs {
j, err := v.JsonIndent(prefix, indent)
if err != nil {
return s, err
}
if haveFirst {
s += "\n"
} else {
haveFirst = true
}
s += string(j)
}
return s, nil
}
// XmlString - analogous to mv.Xml()
func (mvs Maps) XmlString() (string, error) {
var s string
for _, v := range mvs {
x, err := v.Xml()
if err != nil {
return s, err
}
s += string(x)
}
return s, nil
}
// XmlStringIndent - analogous to mv.XmlIndent()
func (mvs Maps) XmlStringIndent(prefix, indent string) (string, error) {
var s string
for _, v := range mvs {
x, err := v.XmlIndent(prefix, indent)
if err != nil {
return s, err
}
s += string(x)
}
return s, nil
}
// JsonFile - write Maps to named file as JSON
// Note: the file will be created, if necessary; if it exists it will be truncated.
// If you need to append to a file, open it and use JsonWriter method.
func (mvs Maps) JsonFile(file string, safeEncoding ...bool) error {
var encoding bool
if len(safeEncoding) == 1 {
encoding = safeEncoding[0]
}
s, err := mvs.JsonString(encoding)
if err != nil {
return err
}
fh, err := os.Create(file)
if err != nil {
return err
}
defer fh.Close()
fh.WriteString(s)
return nil
}
// JsonFileIndent - write Maps to named file as pretty JSON
// Note: the file will be created, if necessary; if it exists it will be truncated.
// If you need to append to a file, open it and use JsonIndentWriter method.
func (mvs Maps) JsonFileIndent(file, prefix, indent string, safeEncoding ...bool) error {
var encoding bool
if len(safeEncoding) == 1 {
encoding = safeEncoding[0]
}
s, err := mvs.JsonStringIndent(prefix, indent, encoding)
if err != nil {
return err
}
fh, err := os.Create(file)
if err != nil {
return err
}
defer fh.Close()
fh.WriteString(s)
return nil
}
// XmlFile - write Maps to named file as XML
// Note: the file will be created, if necessary; if it exists it will be truncated.
// If you need to append to a file, open it and use XmlWriter method.
func (mvs Maps) XmlFile(file string) error {
s, err := mvs.XmlString()
if err != nil {
return err
}
fh, err := os.Create(file)
if err != nil {
return err
}
defer fh.Close()
fh.WriteString(s)
return nil
}
// XmlFileIndent - write Maps to named file as pretty XML
// Note: the file will be created,if necessary; if it exists it will be truncated.
// If you need to append to a file, open it and use XmlIndentWriter method.
func (mvs Maps) XmlFileIndent(file, prefix, indent string) error {
s, err := mvs.XmlStringIndent(prefix, indent)
if err != nil {
return err
}
fh, err := os.Create(file)
if err != nil {
return err
}
defer fh.Close()
fh.WriteString(s)
return nil
}
package mxj
func FuzzMapXml(data []byte) int {
m, err := NewMapXml(data)
if err != nil {
return 0
}
_, err = m.Xml()
if err != nil {
return 0
}
return 1
}
// gob.go - Encode/Decode a Map into a gob object.
package mxj
import (
"bytes"
"encoding/gob"
)
// NewMapGob returns a Map value for a gob object that has been
// encoded from a map[string]interface{} (or compatible type) value.
// It is intended to provide symmetric handling of Maps that have
// been encoded using mv.Gob.
func NewMapGob(gobj []byte) (Map, error) {
m := make(map[string]interface{}, 0)
if len(gobj) == 0 {
return m, nil
}
r := bytes.NewReader(gobj)
dec := gob.NewDecoder(r)
if err := dec.Decode(&m); err != nil {
return m, err
}
return m, nil
}
// Gob returns a gob-encoded value for the Map 'mv'.
func (mv Map) Gob() ([]byte, error) {
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
if err := enc.Encode(map[string]interface{}(mv)); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// Copyright 2012-2014 Charles Banning. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file
package mxj
import (
"bytes"
"encoding/json"
"fmt"
"io"
"time"
)
// ------------------------------ write JSON -----------------------
// Just a wrapper on json.Marshal.
// If option safeEncoding is'true' then safe encoding of '<', '>' and '&'
// is preserved. (see encoding/json#Marshal, encoding/json#Encode)
func (mv Map) Json(safeEncoding ...bool) ([]byte, error) {
var s bool
if len(safeEncoding) == 1 {
s = safeEncoding[0]
}
b, err := json.Marshal(mv)
if !s {
b = bytes.Replace(b, []byte("\\u003c"), []byte("<"), -1)
b = bytes.Replace(b, []byte("\\u003e"), []byte(">"), -1)
b = bytes.Replace(b, []byte("\\u0026"), []byte("&"), -1)
}
return b, err
}
// Just a wrapper on json.MarshalIndent.
// If option safeEncoding is'true' then safe encoding of '<' , '>' and '&'
// is preserved. (see encoding/json#Marshal, encoding/json#Encode)
func (mv Map) JsonIndent(prefix, indent string, safeEncoding ...bool) ([]byte, error) {
var s bool
if len(safeEncoding) == 1 {
s = safeEncoding[0]
}
b, err := json.MarshalIndent(mv, prefix, indent)
if !s {
b = bytes.Replace(b, []byte("\\u003c"), []byte("<"), -1)
b = bytes.Replace(b, []byte("\\u003e"), []byte(">"), -1)
b = bytes.Replace(b, []byte("\\u0026"), []byte("&"), -1)
}
return b, err
}
// The following implementation is provided for symmetry with NewMapJsonReader[Raw]
// The names will also provide a key for the number of return arguments.
// Writes the Map as JSON on the Writer.
// If 'safeEncoding' is 'true', then "safe" encoding of '<', '>' and '&' is preserved.
func (mv Map) JsonWriter(jsonWriter io.Writer, safeEncoding ...bool) error {
b, err := mv.Json(safeEncoding...)
if err != nil {
return err
}
_, err = jsonWriter.Write(b)
return err
}
// Writes the Map as JSON on the Writer. []byte is the raw JSON that was written.
// If 'safeEncoding' is 'true', then "safe" encoding of '<', '>' and '&' is preserved.
func (mv Map) JsonWriterRaw(jsonWriter io.Writer, safeEncoding ...bool) ([]byte, error) {
b, err := mv.Json(safeEncoding...)
if err != nil {
return b, err
}
_, err = jsonWriter.Write(b)
return b, err
}
// Writes the Map as pretty JSON on the Writer.
// If 'safeEncoding' is 'true', then "safe" encoding of '<', '>' and '&' is preserved.
func (mv Map) JsonIndentWriter(jsonWriter io.Writer, prefix, indent string, safeEncoding ...bool) error {
b, err := mv.JsonIndent(prefix, indent, safeEncoding...)
if err != nil {
return err
}
_, err = jsonWriter.Write(b)
return err
}
// Writes the Map as pretty JSON on the Writer. []byte is the raw JSON that was written.
// If 'safeEncoding' is 'true', then "safe" encoding of '<', '>' and '&' is preserved.
func (mv Map) JsonIndentWriterRaw(jsonWriter io.Writer, prefix, indent string, safeEncoding ...bool) ([]byte, error) {
b, err := mv.JsonIndent(prefix, indent, safeEncoding...)
if err != nil {
return b, err
}
_, err = jsonWriter.Write(b)
return b, err
}
// --------------------------- read JSON -----------------------------
// Decode numericvalues as json.Number type Map values - see encoding/json#Number.
// NOTE: this is for decoding JSON into a Map with NewMapJson(), NewMapJsonReader(),
// etc.; it does not affect NewMapXml(), etc. The XML encoders mv.Xml() and mv.XmlIndent()
// do recognize json.Number types; a JSON object can be decoded to a Map with json.Number
// value types and the resulting Map can be correctly encoded into a XML object.
var JsonUseNumber bool
// Just a wrapper on json.Unmarshal
// Converting JSON to XML is a simple as:
// ...
// mapVal, merr := mxj.NewMapJson(jsonVal)
// if merr != nil {
// // handle error
// }
// xmlVal, xerr := mapVal.Xml()
// if xerr != nil {
// // handle error
// }
// NOTE: as a special case, passing a list, e.g., [{"some-null-value":"", "a-non-null-value":"bar"}],
// will be interpreted as having the root key 'object' prepended - {"object":[ ... ]} - to unmarshal to a Map.
// See mxj/j2x/j2x_test.go.
func NewMapJson(jsonVal []byte) (Map, error) {
// empty or nil begets empty
if len(jsonVal) == 0 {
m := make(map[string]interface{}, 0)
return m, nil
}
// handle a goofy case ...
if jsonVal[0] == '[' {
jsonVal = []byte(`{"object":` + string(jsonVal) + `}`)
}
m := make(map[string]interface{})
// err := json.Unmarshal(jsonVal, &m)
buf := bytes.NewReader(jsonVal)
dec := json.NewDecoder(buf)
if JsonUseNumber {
dec.UseNumber()
}
err := dec.Decode(&m)
return m, err
}
// Retrieve a Map value from an io.Reader.
// NOTE: The raw JSON off the reader is buffered to []byte using a ByteReader. If the io.Reader is an
// os.File, there may be significant performance impact. If the io.Reader is wrapping a []byte
// value in-memory, however, such as http.Request.Body you CAN use it to efficiently unmarshal
// a JSON object.
func NewMapJsonReader(jsonReader io.Reader) (Map, error) {
jb, err := getJson(jsonReader)
if err != nil || len(*jb) == 0 {
return nil, err
}
// Unmarshal the 'presumed' JSON string
return NewMapJson(*jb)
}
// Retrieve a Map value and raw JSON - []byte - from an io.Reader.
// NOTE: The raw JSON off the reader is buffered to []byte using a ByteReader. If the io.Reader is an
// os.File, there may be significant performance impact. If the io.Reader is wrapping a []byte
// value in-memory, however, such as http.Request.Body you CAN use it to efficiently unmarshal
// a JSON object and retrieve the raw JSON in a single call.
func NewMapJsonReaderRaw(jsonReader io.Reader) (Map, []byte, error) {
jb, err := getJson(jsonReader)
if err != nil || len(*jb) == 0 {
return nil, *jb, err
}
// Unmarshal the 'presumed' JSON string
m, merr := NewMapJson(*jb)
return m, *jb, merr
}
// Pull the next JSON string off the stream: just read from first '{' to its closing '}'.
// Returning a pointer to the slice saves 16 bytes - maybe unnecessary, but internal to package.
func getJson(rdr io.Reader) (*[]byte, error) {
bval := make([]byte, 1)
jb := make([]byte, 0)
var inQuote, inJson bool
var parenCnt int
var previous byte
// scan the input for a matched set of {...}
// json.Unmarshal will handle syntax checking.
for {
_, err := rdr.Read(bval)
if err != nil {
if err == io.EOF && inJson && parenCnt > 0 {
return &jb, fmt.Errorf("no closing } for JSON string: %s", string(jb))
}
return &jb, err
}
switch bval[0] {
case '{':
if !inQuote {
parenCnt++
inJson = true
}
case '}':
if !inQuote {
parenCnt--
}
if parenCnt < 0 {
return nil, fmt.Errorf("closing } without opening {: %s", string(jb))
}
case '"':
if inQuote {
if previous == '\\' {
break
}
inQuote = false
} else {
inQuote = true
}
case '\n', '\r', '\t', ' ':
if !inQuote {
continue
}
}
if inJson {
jb = append(jb, bval[0])
if parenCnt == 0 {
break
}
}
previous = bval[0]
}
return &jb, nil
}
// ------------------------------- JSON Reader handler via Map values -----------------------
// Default poll delay to keep Handler from spinning on an open stream
// like sitting on os.Stdin waiting for imput.
var jhandlerPollInterval = time.Duration(1e6)
// While unnecessary, we make HandleJsonReader() have the same signature as HandleXmlReader().
// This avoids treating one or other as a special case and discussing the underlying stdlib logic.
// Bulk process JSON using handlers that process a Map value.
// 'rdr' is an io.Reader for the JSON (stream).
// 'mapHandler' is the Map processing handler. Return of 'false' stops io.Reader processing.
// 'errHandler' is the error processor. Return of 'false' stops io.Reader processing and returns the error.
// Note: mapHandler() and errHandler() calls are blocking, so reading and processing of messages is serialized.
// This means that you can stop reading the file on error or after processing a particular message.
// To have reading and handling run concurrently, pass argument to a go routine in handler and return 'true'.
func HandleJsonReader(jsonReader io.Reader, mapHandler func(Map) bool, errHandler func(error) bool) error {
var n int
for {
m, merr := NewMapJsonReader(jsonReader)
n++
// handle error condition with errhandler
if merr != nil && merr != io.EOF {
merr = fmt.Errorf("[jsonReader: %d] %s", n, merr.Error())
if ok := errHandler(merr); !ok {
// caused reader termination
return merr
}
continue
}
// pass to maphandler
if len(m) != 0 {
if ok := mapHandler(m); !ok {
break
}
} else if merr != io.EOF {
<-time.After(jhandlerPollInterval)
}
if merr == io.EOF {
break
}
}
return nil
}
// Bulk process JSON using handlers that process a Map value and the raw JSON.
// 'rdr' is an io.Reader for the JSON (stream).
// 'mapHandler' is the Map and raw JSON - []byte - processor. Return of 'false' stops io.Reader processing.
// 'errHandler' is the error and raw JSON processor. Return of 'false' stops io.Reader processing and returns the error.
// Note: mapHandler() and errHandler() calls are blocking, so reading and processing of messages is serialized.
// This means that you can stop reading the file on error or after processing a particular message.
// To have reading and handling run concurrently, pass argument(s) to a go routine in handler and return 'true'.
func HandleJsonReaderRaw(jsonReader io.Reader, mapHandler func(Map, []byte) bool, errHandler func(error, []byte) bool) error {
var n int
for {
m, raw, merr := NewMapJsonReaderRaw(jsonReader)
n++
// handle error condition with errhandler
if merr != nil && merr != io.EOF {
merr = fmt.Errorf("[jsonReader: %d] %s", n, merr.Error())
if ok := errHandler(merr, raw); !ok {
// caused reader termination
return merr
}
continue
}
// pass to maphandler
if len(m) != 0 {
if ok := mapHandler(m, raw); !ok {
break
}
} else if merr != io.EOF {
<-time.After(jhandlerPollInterval)
}
if merr == io.EOF {
break
}
}
return nil
}
// Copyright 2012-2014 Charles Banning. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file
// keyvalues.go: Extract values from an arbitrary XML doc. Tag path can include wildcard characters.
package mxj
import (
"errors"
"fmt"
"strconv"
"strings"
)
// ----------------------------- get everything FOR a single key -------------------------
const (
minArraySize = 32
)
var defaultArraySize int = minArraySize
// SetArraySize adjust the buffers for expected number of values to return from ValuesForKey() and ValuesForPath().
// This can have the effect of significantly reducing memory allocation-copy functions for large data sets.
// Returns the initial buffer size.
func SetArraySize(size int) int {
if size > minArraySize {
defaultArraySize = size
} else {
defaultArraySize = minArraySize
}
return defaultArraySize
}
// ValuesForKey return all values in Map, 'mv', associated with a 'key'. If len(returned_values) == 0, then no match.
// On error, the returned slice is 'nil'. NOTE: 'key' can be wildcard, "*".
// 'subkeys' (optional) are "key:val[:type]" strings representing attributes or elements in a list.
// - By default 'val' is of type string. "key:val:bool" and "key:val:float" to coerce them.
// - For attributes prefix the label with the attribute prefix character, by default a
// hyphen, '-', e.g., "-seq:3". (See SetAttrPrefix function.)
// - If the 'key' refers to a list, then "key:value" could select a list member of the list.
// - The subkey can be wildcarded - "key:*" - to require that it's there with some value.
// - If a subkey is preceeded with the '!' character, the key:value[:type] entry is treated as an
// exclusion critera - e.g., "!author:William T. Gaddis".
// - If val contains ":" symbol, use SetFieldSeparator to a unused symbol, perhaps "|".
func (mv Map) ValuesForKey(key string, subkeys ...string) ([]interface{}, error) {
m := map[string]interface{}(mv)
var subKeyMap map[string]interface{}
if len(subkeys) > 0 {
var err error
subKeyMap, err = getSubKeyMap(subkeys...)
if err != nil {
return nil, err
}
}
ret := make([]interface{}, 0, defaultArraySize)
var cnt int
hasKey(m, key, &ret, &cnt, subKeyMap)
return ret[:cnt], nil
}
var KeyNotExistError = errors.New("Key does not exist")
// ValueForKey is a wrapper on ValuesForKey. It returns the first member of []interface{}, if any.
// If there is no value, "nil, nil" is returned.
func (mv Map) ValueForKey(key string, subkeys ...string) (interface{}, error) {
vals, err := mv.ValuesForKey(key, subkeys...)
if err != nil {
return nil, err
}
if len(vals) == 0 {
return nil, KeyNotExistError
}
return vals[0], nil
}
// hasKey - if the map 'key' exists append it to array
// if it doesn't do nothing except scan array and map values
func hasKey(iv interface{}, key string, ret *[]interface{}, cnt *int, subkeys map[string]interface{}) {
// func hasKey(iv interface{}, key string, ret *[]interface{}, subkeys map[string]interface{}) {
switch iv.(type) {
case map[string]interface{}:
vv := iv.(map[string]interface{})
// see if the current value is of interest
if v, ok := vv[key]; ok {
switch v.(type) {
case map[string]interface{}:
if hasSubKeys(v, subkeys) {
*ret = append(*ret, v)
*cnt++
}
case []interface{}:
for _, av := range v.([]interface{}) {
if hasSubKeys(av, subkeys) {
*ret = append(*ret, av)
*cnt++
}
}
default:
if len(subkeys) == 0 {
*ret = append(*ret, v)
*cnt++
}
}
}
// wildcard case
if key == "*" {
for _, v := range vv {
switch v.(type) {
case map[string]interface{}:
if hasSubKeys(v, subkeys) {
*ret = append(*ret, v)
*cnt++
}
case []interface{}:
for _, av := range v.([]interface{}) {
if hasSubKeys(av, subkeys) {
*ret = append(*ret, av)
*cnt++
}
}
default:
if len(subkeys) == 0 {
*ret = append(*ret, v)
*cnt++
}
}
}
}
// scan the rest
for _, v := range vv {
hasKey(v, key, ret, cnt, subkeys)
}
case []interface{}:
for _, v := range iv.([]interface{}) {
hasKey(v, key, ret, cnt, subkeys)
}
}
}
// ----------------------- get everything for a node in the Map ---------------------------
// Allow indexed arrays in "path" specification. (Request from Abhijit Kadam - abhijitk100@gmail.com.)
// 2014.04.28 - implementation note.
// Implemented as a wrapper of (old)ValuesForPath() because we need look-ahead logic to handle expansion
// of wildcards and unindexed arrays. Embedding such logic into valuesForKeyPath() would have made the
// code much more complicated; this wrapper is straightforward, easy to debug, and doesn't add significant overhead.
// ValuesForPatb retrieves all values for a path from the Map. If len(returned_values) == 0, then no match.
// On error, the returned array is 'nil'.
// 'path' is a dot-separated path of key values.
// - If a node in the path is '*', then everything beyond is walked.
// - 'path' can contain indexed array references, such as, "*.data[1]" and "msgs[2].data[0].field" -
// even "*[2].*[0].field".
// 'subkeys' (optional) are "key:val[:type]" strings representing attributes or elements in a list.
// - By default 'val' is of type string. "key:val:bool" and "key:val:float" to coerce them.
// - For attributes prefix the label with the attribute prefix character, by default a
// hyphen, '-', e.g., "-seq:3". (See SetAttrPrefix function.)
// - If the 'path' refers to a list, then "tag:value" would return member of the list.
// - The subkey can be wildcarded - "key:*" - to require that it's there with some value.
// - If a subkey is preceeded with the '!' character, the key:value[:type] entry is treated as an
// exclusion critera - e.g., "!author:William T. Gaddis".
// - If val contains ":" symbol, use SetFieldSeparator to a unused symbol, perhaps "|".
func (mv Map) ValuesForPath(path string, subkeys ...string) ([]interface{}, error) {
// If there are no array indexes in path, use legacy ValuesForPath() logic.
if strings.Index(path, "[") < 0 {
return mv.oldValuesForPath(path, subkeys...)
}
var subKeyMap map[string]interface{}
if len(subkeys) > 0 {
var err error
subKeyMap, err = getSubKeyMap(subkeys...)
if err != nil {
return nil, err
}
}
keys, kerr := parsePath(path)
if kerr != nil {
return nil, kerr
}
vals, verr := valuesForArray(keys, mv)
if verr != nil {
return nil, verr // Vals may be nil, but return empty array.
}
// Need to handle subkeys ... only return members of vals that satisfy conditions.
retvals := make([]interface{}, 0)
for _, v := range vals {
if hasSubKeys(v, subKeyMap) {
retvals = append(retvals, v)
}
}
return retvals, nil
}
func valuesForArray(keys []*key, m Map) ([]interface{}, error) {
var tmppath string
var haveFirst bool
var vals []interface{}
var verr error
lastkey := len(keys) - 1
for i := 0; i <= lastkey; i++ {
if !haveFirst {
tmppath = keys[i].name
haveFirst = true
} else {
tmppath += "." + keys[i].name
}
// Look-ahead: explode wildcards and unindexed arrays.
// Need to handle un-indexed list recursively:
// e.g., path is "stuff.data[0]" rather than "stuff[0].data[0]".
// Need to treat it as "stuff[0].data[0]", "stuff[1].data[0]", ...
if !keys[i].isArray && i < lastkey && keys[i+1].isArray {
// Can't pass subkeys because we may not be at literal end of path.
vv, vverr := m.oldValuesForPath(tmppath)
if vverr != nil {
return nil, vverr
}
for _, v := range vv {
// See if we can walk the value.
am, ok := v.(map[string]interface{})
if !ok {
continue
}
// Work the backend.
nvals, nvalserr := valuesForArray(keys[i+1:], Map(am))
if nvalserr != nil {
return nil, nvalserr
}
vals = append(vals, nvals...)
}
break // have recursed the whole path - return
}
if keys[i].isArray || i == lastkey {
// Don't pass subkeys because may not be at literal end of path.
vals, verr = m.oldValuesForPath(tmppath)
} else {
continue
}
if verr != nil {
return nil, verr
}
if i == lastkey && !keys[i].isArray {
break
}
// Now we're looking at an array - supposedly.
// Is index in range of vals?
if len(vals) <= keys[i].position {
vals = nil
break
}
// Return the array member of interest, if at end of path.
if i == lastkey {
vals = vals[keys[i].position:(keys[i].position + 1)]
break
}
// Extract the array member of interest.
am := vals[keys[i].position:(keys[i].position + 1)]
// must be a map[string]interface{} value so we can keep walking the path
amm, ok := am[0].(map[string]interface{})
if !ok {
vals = nil
break
}
m = Map(amm)
haveFirst = false
}
return vals, nil
}
type key struct {
name string
isArray bool
position int
}
func parsePath(s string) ([]*key, error) {
keys := strings.Split(s, ".")
ret := make([]*key, 0)
for i := 0; i < len(keys); i++ {
if keys[i] == "" {
continue
}
newkey := new(key)
if strings.Index(keys[i], "[") < 0 {
newkey.name = keys[i]
ret = append(ret, newkey)
continue
}
p := strings.Split(keys[i], "[")
newkey.name = p[0]
p = strings.Split(p[1], "]")
if p[0] == "" { // no right bracket
return nil, fmt.Errorf("no right bracket on key index: %s", keys[i])
}
// convert p[0] to a int value
pos, nerr := strconv.ParseInt(p[0], 10, 32)
if nerr != nil {
return nil, fmt.Errorf("cannot convert index to int value: %s", p[0])
}
newkey.position = int(pos)
newkey.isArray = true
ret = append(ret, newkey)
}
return ret, nil
}
// legacy ValuesForPath() - now wrapped to handle special case of indexed arrays in 'path'.
func (mv Map) oldValuesForPath(path string, subkeys ...string) ([]interface{}, error) {
m := map[string]interface{}(mv)
var subKeyMap map[string]interface{}
if len(subkeys) > 0 {
var err error
subKeyMap, err = getSubKeyMap(subkeys...)
if err != nil {
return nil, err
}
}
keys := strings.Split(path, ".")
if keys[len(keys)-1] == "" {
keys = keys[:len(keys)-1]
}
ivals := make([]interface{}, 0, defaultArraySize)
var cnt int
valuesForKeyPath(&ivals, &cnt, m, keys, subKeyMap)
return ivals[:cnt], nil
}
func valuesForKeyPath(ret *[]interface{}, cnt *int, m interface{}, keys []string, subkeys map[string]interface{}) {
lenKeys := len(keys)
// load 'm' values into 'ret'
// expand any lists
if lenKeys == 0 {
switch m.(type) {
case map[string]interface{}:
if subkeys != nil {
if ok := hasSubKeys(m, subkeys); !ok {
return
}
}
*ret = append(*ret, m)
*cnt++
case []interface{}:
for i, v := range m.([]interface{}) {
if subkeys != nil {
if ok := hasSubKeys(v, subkeys); !ok {
continue // only load list members with subkeys
}
}
*ret = append(*ret, (m.([]interface{}))[i])
*cnt++
}
default:
if subkeys != nil {
return // must be map[string]interface{} if there are subkeys
}
*ret = append(*ret, m)
*cnt++
}
return
}
// key of interest
key := keys[0]
switch key {
case "*": // wildcard - scan all values
switch m.(type) {
case map[string]interface{}:
for _, v := range m.(map[string]interface{}) {
// valuesForKeyPath(ret, v, keys[1:], subkeys)
valuesForKeyPath(ret, cnt, v, keys[1:], subkeys)
}
case []interface{}:
for _, v := range m.([]interface{}) {
switch v.(type) {
// flatten out a list of maps - keys are processed
case map[string]interface{}:
for _, vv := range v.(map[string]interface{}) {
// valuesForKeyPath(ret, vv, keys[1:], subkeys)
valuesForKeyPath(ret, cnt, vv, keys[1:], subkeys)
}
default:
// valuesForKeyPath(ret, v, keys[1:], subkeys)
valuesForKeyPath(ret, cnt, v, keys[1:], subkeys)
}
}
}
default: // key - must be map[string]interface{}
switch m.(type) {
case map[string]interface{}:
if v, ok := m.(map[string]interface{})[key]; ok {
// valuesForKeyPath(ret, v, keys[1:], subkeys)
valuesForKeyPath(ret, cnt, v, keys[1:], subkeys)
}
case []interface{}: // may be buried in list
for _, v := range m.([]interface{}) {
switch v.(type) {
case map[string]interface{}:
if vv, ok := v.(map[string]interface{})[key]; ok {
// valuesForKeyPath(ret, vv, keys[1:], subkeys)
valuesForKeyPath(ret, cnt, vv, keys[1:], subkeys)
}
}
}
}
}
}
// hasSubKeys() - interface{} equality works for string, float64, bool
// 'v' must be a map[string]interface{} value to have subkeys
// 'a' can have k:v pairs with v.(string) == "*", which is treated like a wildcard.
func hasSubKeys(v interface{}, subkeys map[string]interface{}) bool {
if len(subkeys) == 0 {
return true
}
switch v.(type) {
case map[string]interface{}:
// do all subKey name:value pairs match?
mv := v.(map[string]interface{})
for skey, sval := range subkeys {
isNotKey := false
if skey[:1] == "!" { // a NOT-key
skey = skey[1:]
isNotKey = true
}
vv, ok := mv[skey]
if !ok { // key doesn't exist
if isNotKey { // key not there, but that's what we want
if kv, ok := sval.(string); ok && kv == "*" {
continue
}
}
return false
}
// wildcard check
if kv, ok := sval.(string); ok && kv == "*" {
if isNotKey { // key is there, and we don't want it
return false
}
continue
}
switch sval.(type) {
case string:
if s, ok := vv.(string); ok && s == sval.(string) {
if isNotKey {
return false
}
continue
}
case bool:
if b, ok := vv.(bool); ok && b == sval.(bool) {
if isNotKey {
return false
}
continue
}
case float64:
if f, ok := vv.(float64); ok && f == sval.(float64) {
if isNotKey {
return false
}
continue
}
}
// key there but didn't match subkey value
if isNotKey { // that's what we want
continue
}
return false
}
// all subkeys matched
return true
}
// not a map[string]interface{} value, can't have subkeys
return false
}
// Generate map of key:value entries as map[string]string.
// 'kv' arguments are "name:value" pairs: attribute keys are designated with prepended hyphen, '-'.
// If len(kv) == 0, the return is (nil, nil).
func getSubKeyMap(kv ...string) (map[string]interface{}, error) {
if len(kv) == 0 {
return nil, nil
}
m := make(map[string]interface{}, 0)
for _, v := range kv {
vv := strings.Split(v, fieldSep)
switch len(vv) {
case 2:
m[vv[0]] = interface{}(vv[1])
case 3:
switch vv[2] {
case "string", "char", "text":
m[vv[0]] = interface{}(vv[1])
case "bool", "boolean":
// ParseBool treats "1"==true & "0"==false
b, err := strconv.ParseBool(vv[1])
if err != nil {
return nil, fmt.Errorf("can't convert subkey value to bool: %s", vv[1])
}
m[vv[0]] = interface{}(b)
case "float", "float64", "num", "number", "numeric":
f, err := strconv.ParseFloat(vv[1], 64)
if err != nil {
return nil, fmt.Errorf("can't convert subkey value to float: %s", vv[1])
}
m[vv[0]] = interface{}(f)
default:
return nil, fmt.Errorf("unknown subkey conversion spec: %s", v)
}
default:
return nil, fmt.Errorf("unknown subkey spec: %s", v)
}
}
return m, nil
}
// ------------------------------- END of valuesFor ... ----------------------------
// ----------------------- locate where a key value is in the tree -------------------
//----------------------------- find all paths to a key --------------------------------
// PathsForKey returns all paths through Map, 'mv', (in dot-notation) that terminate with the specified key.
// Results can be used with ValuesForPath.
func (mv Map) PathsForKey(key string) []string {
m := map[string]interface{}(mv)
breadbasket := make(map[string]bool, 0)
breadcrumbs := ""
hasKeyPath(breadcrumbs, m, key, breadbasket)
if len(breadbasket) == 0 {
return nil
}
// unpack map keys to return
res := make([]string, len(breadbasket))
var i int
for k := range breadbasket {
res[i] = k
i++
}
return res
}
// PathForKeyShortest extracts the shortest path from all possible paths - from PathsForKey() - in Map, 'mv'..
// Paths are strings using dot-notation.
func (mv Map) PathForKeyShortest(key string) string {
paths := mv.PathsForKey(key)
lp := len(paths)
if lp == 0 {
return ""
}
if lp == 1 {
return paths[0]
}
shortest := paths[0]
shortestLen := len(strings.Split(shortest, "."))
for i := 1; i < len(paths); i++ {
vlen := len(strings.Split(paths[i], "."))
if vlen < shortestLen {
shortest = paths[i]
shortestLen = vlen
}
}
return shortest
}
// hasKeyPath - if the map 'key' exists append it to KeyPath.path and increment KeyPath.depth
// This is really just a breadcrumber that saves all trails that hit the prescribed 'key'.
func hasKeyPath(crumbs string, iv interface{}, key string, basket map[string]bool) {
switch iv.(type) {
case map[string]interface{}:
vv := iv.(map[string]interface{})
if _, ok := vv[key]; ok {
// create a new breadcrumb, intialized with the one we have
var nbc string
if crumbs == "" {
nbc = key
} else {
nbc = crumbs + "." + key
}
basket[nbc] = true
}
// walk on down the path, key could occur again at deeper node
for k, v := range vv {
// create a new breadcrumb, intialized with the one we have
var nbc string
if crumbs == "" {
nbc = k
} else {
nbc = crumbs + "." + k
}
hasKeyPath(nbc, v, key, basket)
}
case []interface{}:
// crumb-trail doesn't change, pass it on
for _, v := range iv.([]interface{}) {
hasKeyPath(crumbs, v, key, basket)
}
}
}
var PathNotExistError = errors.New("Path does not exist")
// ValueForPath wraps ValuesFor Path and returns the first value returned.
// If no value is found it returns 'nil' and PathNotExistError.
func (mv Map) ValueForPath(path string) (interface{}, error) {
vals, err := mv.ValuesForPath(path)
if err != nil {
return nil, err
}
if len(vals) == 0 {
return nil, PathNotExistError
}
return vals[0], nil
}
// ValuesForPathString returns the first found value for the path as a string.
func (mv Map) ValueForPathString(path string) (string, error) {
vals, err := mv.ValuesForPath(path)
if err != nil {
return "", err
}
if len(vals) == 0 {
return "", errors.New("ValueForPath: path not found")
}
val := vals[0]
return fmt.Sprintf("%v", val), nil
}
// ValueOrEmptyForPathString returns the first found value for the path as a string.
// If the path is not found then it returns an empty string.
func (mv Map) ValueOrEmptyForPathString(path string) string {
str, _ := mv.ValueForPathString(path)
return str
}
package mxj
// leafnode.go - return leaf nodes with paths and values for the Map
// inspired by: https://groups.google.com/forum/#!topic/golang-nuts/3JhuVKRuBbw
import (
"strconv"
"strings"
)
const (
NoAttributes = true // suppress LeafNode values that are attributes
)
// LeafNode - a terminal path value in a Map.
// For XML Map values it represents an attribute or simple element value - of type
// string unless Map was created using Cast flag. For JSON Map values it represents
// a string, numeric, boolean, or null value.
type LeafNode struct {
Path string // a dot-notation representation of the path with array subscripting
Value interface{} // the value at the path termination
}
// LeafNodes - returns an array of all LeafNode values for the Map.
// The option no_attr argument suppresses attribute values (keys with prepended hyphen, '-')
// as well as the "#text" key for the associated simple element value.
//
// PrependAttrWithHypen(false) will result in attributes having .attr-name as
// terminal node in 'path' while the path for the element value, itself, will be
// the base path w/o "#text".
//
// LeafUseDotNotation(true) causes list members to be identified using ".N" syntax
// rather than "[N]" syntax.
func (mv Map) LeafNodes(no_attr ...bool) []LeafNode {
var a bool
if len(no_attr) == 1 {
a = no_attr[0]
}
l := make([]LeafNode, 0)
getLeafNodes("", "", map[string]interface{}(mv), &l, a)
return l
}
func getLeafNodes(path, node string, mv interface{}, l *[]LeafNode, noattr bool) {
// if stripping attributes, then also strip "#text" key
if !noattr || node != textK {
if path != "" && node[:1] != "[" {
path += "."
}
path += node
}
switch mv.(type) {
case map[string]interface{}:
for k, v := range mv.(map[string]interface{}) {
// if noattr && k[:1] == "-" {
if noattr && len(attrPrefix) > 0 && strings.Index(k, attrPrefix) == 0 {
continue
}
getLeafNodes(path, k, v, l, noattr)
}
case []interface{}:
for i, v := range mv.([]interface{}) {
if useDotNotation {
getLeafNodes(path, strconv.Itoa(i), v, l, noattr)
} else {
getLeafNodes(path, "["+strconv.Itoa(i)+"]", v, l, noattr)
}
}
default:
// can't walk any further, so create leaf
n := LeafNode{path, mv}
*l = append(*l, n)
}
}
// LeafPaths - all paths that terminate in LeafNode values.
func (mv Map) LeafPaths(no_attr ...bool) []string {
ln := mv.LeafNodes()
ss := make([]string, len(ln))
for i := 0; i < len(ln); i++ {
ss[i] = ln[i].Path
}
return ss
}
// LeafValues - all terminal values in the Map.
func (mv Map) LeafValues(no_attr ...bool) []interface{} {
ln := mv.LeafNodes()
vv := make([]interface{}, len(ln))
for i := 0; i < len(ln); i++ {
vv[i] = ln[i].Value
}
return vv
}
// ====================== utilities ======================
// https://groups.google.com/forum/#!topic/golang-nuts/pj0C5IrZk4I
var useDotNotation bool
// LeafUseDotNotation sets a flag that list members in LeafNode paths
// should be identified using ".N" syntax rather than the default "[N]"
// syntax. Calling LeafUseDotNotation with no arguments toggles the
// flag on/off; otherwise, the argument sets the flag value 'true'/'false'.
func LeafUseDotNotation(b ...bool) {
if len(b) == 0 {
useDotNotation = !useDotNotation
return
}
useDotNotation = b[0]
}
// Copyright 2016 Charles Banning. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file
// misc.go - mimic functions (+others) called out in:
// https://groups.google.com/forum/#!topic/golang-nuts/jm_aGsJNbdQ
// Primarily these methods let you retrive XML structure information.
package mxj
import (
"fmt"
"sort"
"strings"
)
// Return the root element of the Map. If there is not a single key in Map,
// then an error is returned.
func (mv Map) Root() (string, error) {
mm := map[string]interface{}(mv)
if len(mm) != 1 {
return "", fmt.Errorf("Map does not have singleton root. Len: %d.", len(mm))
}
for k, _ := range mm {
return k, nil
}
return "", nil
}
// If the path is an element with sub-elements, return a list of the sub-element
// keys. (The list is alphabeticly sorted.) NOTE: Map keys that are prefixed with
// '-', a hyphen, are considered attributes; see m.Attributes(path).
func (mv Map) Elements(path string) ([]string, error) {
e, err := mv.ValueForPath(path)
if err != nil {
return nil, err
}
switch e.(type) {
case map[string]interface{}:
ee := e.(map[string]interface{})
elems := make([]string, len(ee))
var i int
for k, _ := range ee {
if len(attrPrefix) > 0 && strings.Index(k, attrPrefix) == 0 {
continue // skip attributes
}
elems[i] = k
i++
}
elems = elems[:i]
// alphabetic sort keeps things tidy
sort.Strings(elems)
return elems, nil
}
return nil, fmt.Errorf("no elements for path: %s", path)
}
// If the path is an element with attributes, return a list of the attribute
// keys. (The list is alphabeticly sorted.) NOTE: Map keys that are not prefixed with
// '-', a hyphen, are not treated as attributes; see m.Elements(path). Also, if the
// attribute prefix is "" - SetAttrPrefix("") or PrependAttrWithHyphen(false) - then
// there are no identifiable attributes.
func (mv Map) Attributes(path string) ([]string, error) {
a, err := mv.ValueForPath(path)
if err != nil {
return nil, err
}
switch a.(type) {
case map[string]interface{}:
aa := a.(map[string]interface{})
attrs := make([]string, len(aa))
var i int
for k, _ := range aa {
if len(attrPrefix) == 0 || strings.Index(k, attrPrefix) != 0 {
continue // skip non-attributes
}
attrs[i] = k[len(attrPrefix):]
i++
}
attrs = attrs[:i]
// alphabetic sort keeps things tidy
sort.Strings(attrs)
return attrs, nil
}
return nil, fmt.Errorf("no attributes for path: %s", path)
}
// mxj - A collection of map[string]interface{} and associated XML and JSON utilities.
// Copyright 2012-2014 Charles Banning. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file
package mxj
import (
"fmt"
"sort"
)
const (
Cast = true // for clarity - e.g., mxj.NewMapXml(doc, mxj.Cast)
SafeEncoding = true // ditto - e.g., mv.Json(mxj.SafeEncoding)
)
type Map map[string]interface{}
// Allocate a Map.
func New() Map {
m := make(map[string]interface{}, 0)
return m
}
// Cast a Map to map[string]interface{}
func (mv Map) Old() map[string]interface{} {
return mv
}
// Return a copy of mv as a newly allocated Map. If the Map only contains string,
// numeric, map[string]interface{}, and []interface{} values, then it can be thought
// of as a "deep copy." Copying a structure (or structure reference) value is subject
// to the noted restrictions.
// NOTE: If 'mv' includes structure values with, possibly, JSON encoding tags
// then only public fields of the structure are in the new Map - and with
// keys that conform to any encoding tag instructions. The structure itself will
// be represented as a map[string]interface{} value.
func (mv Map) Copy() (Map, error) {
// this is the poor-man's deep copy
// not efficient, but it works
j, jerr := mv.Json()
// must handle, we don't know how mv got built
if jerr != nil {
return nil, jerr
}
return NewMapJson(j)
}
// --------------- StringIndent ... from x2j.WriteMap -------------
// Pretty print a Map.
func (mv Map) StringIndent(offset ...int) string {
return writeMap(map[string]interface{}(mv), true, true, offset...)
}
// Pretty print a Map without the value type information - just key:value entries.
func (mv Map) StringIndentNoTypeInfo(offset ...int) string {
return writeMap(map[string]interface{}(mv), false, true, offset...)
}
// writeMap - dumps the map[string]interface{} for examination.
// 'typeInfo' causes value type to be printed.
// 'offset' is initial indentation count; typically: Write(m).
func writeMap(m interface{}, typeInfo, root bool, offset ...int) string {
var indent int
if len(offset) == 1 {
indent = offset[0]
}
var s string
switch m.(type) {
case []interface{}:
if typeInfo {
s += "[[]interface{}]"
}
for _, v := range m.([]interface{}) {
s += "\n"
for i := 0; i < indent; i++ {
s += " "
}
s += writeMap(v, typeInfo, false, indent+1)
}
case map[string]interface{}:
list := make([][2]string, len(m.(map[string]interface{})))
var n int
for k, v := range m.(map[string]interface{}) {
list[n][0] = k
list[n][1] = writeMap(v, typeInfo, false, indent+1)
n++
}
sort.Sort(mapList(list))
for _, v := range list {
if root {
root = false
} else {
s += "\n"
}
for i := 0; i < indent; i++ {
s += " "
}
s += v[0] + " : " + v[1]
}
default:
if typeInfo {
s += fmt.Sprintf("[%T] %+v", m, m)
} else {
s += fmt.Sprintf("%+v", m)
}
}
return s
}
// ======================== utility ===============
type mapList [][2]string
func (ml mapList) Len() int {
return len(ml)
}
func (ml mapList) Swap(i, j int) {
ml[i], ml[j] = ml[j], ml[i]
}
func (ml mapList) Less(i, j int) bool {
return ml[i][0] <= ml[j][0]
}
// mxj - A collection of map[string]interface{} and associated XML and JSON utilities.
// Copyright 2012-2014, 2018 Charles Banning. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file
// remap.go - build a new Map from the current Map based on keyOld:keyNew mapppings
// keys can use dot-notation, keyOld can use wildcard, '*'
//
// Computational strategy -
// Using the key path - []string - traverse a new map[string]interface{} and
// insert the oldVal as the newVal when we arrive at the end of the path.
// If the type at the end is nil, then that is newVal
// If the type at the end is a singleton (string, float64, bool) an array is created.
// If the type at the end is an array, newVal is just appended.
// If the type at the end is a map, it is inserted if possible or the map value
// is converted into an array if necessary.
package mxj
import (
"errors"
"strings"
)
// (Map)NewMap - create a new Map from data in the current Map.
// 'keypairs' are key mappings "oldKey:newKey" and specify that the current value of 'oldKey'
// should be the value for 'newKey' in the returned Map.
// - 'oldKey' supports dot-notation as described for (Map)ValuesForPath()
// - 'newKey' supports dot-notation but with no wildcards, '*', or indexed arrays
// - "oldKey" is shorthand for the keypair value "oldKey:oldKey"
// - "oldKey:" and ":newKey" are invalid keypair values
// - if 'oldKey' does not exist in the current Map, it is not written to the new Map.
// "null" is not supported unless it is the current Map.
// - see newmap_test.go for several syntax examples
// - mv.NewMap() == mxj.New()
//
// NOTE: "examples/partial.go" shows how to create arbitrary sub-docs of an XML doc.
func (mv Map) NewMap(keypairs ...string) (Map, error) {
n := make(map[string]interface{}, 0)
if len(keypairs) == 0 {
return n, nil
}
// loop through the pairs
var oldKey, newKey string
var path []string
for _, v := range keypairs {
if len(v) == 0 {
continue // just skip over empty keypair arguments
}
// initialize oldKey, newKey and check
vv := strings.Split(v, ":")
if len(vv) > 2 {
return n, errors.New("oldKey:newKey keypair value not valid - " + v)
}
if len(vv) == 1 {
oldKey, newKey = vv[0], vv[0]
} else {
oldKey, newKey = vv[0], vv[1]
}
strings.TrimSpace(oldKey)
strings.TrimSpace(newKey)
if i := strings.Index(newKey, "*"); i > -1 {
return n, errors.New("newKey value cannot contain wildcard character - " + v)
}
if i := strings.Index(newKey, "["); i > -1 {
return n, errors.New("newKey value cannot contain indexed arrays - " + v)
}
if oldKey == "" || newKey == "" {
return n, errors.New("oldKey or newKey is not specified - " + v)
}
// get oldKey value
oldVal, err := mv.ValuesForPath(oldKey)
if err != nil {
return n, err
}
if len(oldVal) == 0 {
continue // oldKey has no value, may not exist in mv
}
// break down path
path = strings.Split(newKey, ".")
if path[len(path)-1] == "" { // ignore a trailing dot in newKey spec
path = path[:len(path)-1]
}
addNewVal(&n, path, oldVal)
}
return n, nil
}
// navigate 'n' to end of path and add val
func addNewVal(n *map[string]interface{}, path []string, val []interface{}) {
// newVal - either singleton or array
var newVal interface{}
if len(val) == 1 {
newVal = val[0] // is type interface{}
} else {
newVal = interface{}(val)
}
// walk to the position of interest, create it if necessary
m := (*n) // initialize map walker
var k string // key for m
lp := len(path) - 1 // when to stop looking
for i := 0; i < len(path); i++ {
k = path[i]
if i == lp {
break
}
var nm map[string]interface{} // holds position of next-map
switch m[k].(type) {
case nil: // need a map for next node in path, so go there
nm = make(map[string]interface{}, 0)
m[k] = interface{}(nm)
m = m[k].(map[string]interface{})
case map[string]interface{}:
// OK - got somewhere to walk to, go there
m = m[k].(map[string]interface{})
case []interface{}:
// add a map and nm points to new map unless there's already
// a map in the array, then nm points there
// The placement of the next value in the array is dependent
// on the sequence of members - could land on a map or a nil
// value first. TODO: how to test this.
a := make([]interface{}, 0)
var foundmap bool
for _, vv := range m[k].([]interface{}) {
switch vv.(type) {
case nil: // doesn't appear that this occurs, need a test case
if foundmap { // use the first one in array
a = append(a, vv)
continue
}
nm = make(map[string]interface{}, 0)
a = append(a, interface{}(nm))
foundmap = true
case map[string]interface{}:
if foundmap { // use the first one in array
a = append(a, vv)
continue
}
nm = vv.(map[string]interface{})
a = append(a, vv)
foundmap = true
default:
a = append(a, vv)
}
}
// no map found in array
if !foundmap {
nm = make(map[string]interface{}, 0)
a = append(a, interface{}(nm))
}
m[k] = interface{}(a) // must insert in map
m = nm
default: // it's a string, float, bool, etc.
aa := make([]interface{}, 0)
nm = make(map[string]interface{}, 0)
aa = append(aa, m[k], nm)
m[k] = interface{}(aa)
m = nm
}
}
// value is nil, array or a singleton of some kind
// initially m.(type) == map[string]interface{}
v := m[k]
switch v.(type) {
case nil: // initialized
m[k] = newVal
case []interface{}:
a := m[k].([]interface{})
a = append(a, newVal)
m[k] = interface{}(a)
default: // v exists:string, float64, bool, map[string]interface, etc.
a := make([]interface{}, 0)
a = append(a, v, newVal)
m[k] = interface{}(a)
}
}
package mxj
import "strings"
// Removes the path.
func (mv Map) Remove(path string) error {
m := map[string]interface{}(mv)
return remove(m, path)
}
func remove(m interface{}, path string) error {
val, err := prevValueByPath(m, path)
if err != nil {
return err
}
lastKey := lastKey(path)
delete(val, lastKey)
return nil
}
// returns the last key of the path.
// lastKey("a.b.c") would had returned "c"
func lastKey(path string) string {
keys := strings.Split(path, ".")
key := keys[len(keys)-1]
return key
}
// returns the path without the last key
// parentPath("a.b.c") whould had returned "a.b"
func parentPath(path string) string {
keys := strings.Split(path, ".")
parentPath := strings.Join(keys[0:len(keys)-1], ".")
return parentPath
}
package mxj
import (
"errors"
"strings"
)
// RenameKey renames a key in a Map.
// It works only for nested maps.
// It doesn't work for cases when the key is in a list.
func (mv Map) RenameKey(path string, newName string) error {
var v bool
var err error
if v, err = mv.Exists(path); err == nil && !v {
return errors.New("RenameKey: path not found: " + path)
} else if err != nil {
return err
}
if v, err = mv.Exists(parentPath(path) + "." + newName); err == nil && v {
return errors.New("RenameKey: key already exists: " + newName)
} else if err != nil {
return err
}
m := map[string]interface{}(mv)
return renameKey(m, path, newName)
}
func renameKey(m interface{}, path string, newName string) error {
val, err := prevValueByPath(m, path)
if err != nil {
return err
}
oldName := lastKey(path)
val[newName] = val[oldName]
delete(val, oldName)
return nil
}
// returns a value which contains a last key in the path
// For example: prevValueByPath("a.b.c", {a{b{c: 3}}}) returns {c: 3}
func prevValueByPath(m interface{}, path string) (map[string]interface{}, error) {
keys := strings.Split(path, ".")
switch mValue := m.(type) {
case map[string]interface{}:
for key, value := range mValue {
if key == keys[0] {
if len(keys) == 1 {
return mValue, nil
} else {
// keep looking for the full path to the key
return prevValueByPath(value, strings.Join(keys[1:], "."))
}
}
}
}
return nil, errors.New("prevValueByPath: didn't find path – " + path)
}
package mxj
import (
"strings"
)
// Sets the value for the path
func (mv Map) SetValueForPath(value interface{}, path string) error {
pathAry := strings.Split(path, ".")
parentPathAry := pathAry[0 : len(pathAry)-1]
parentPath := strings.Join(parentPathAry, ".")
val, err := mv.ValueForPath(parentPath)
if err != nil {
return err
}
if val == nil {
return nil // we just ignore the request if there's no val
}
key := pathAry[len(pathAry)-1]
cVal := val.(map[string]interface{})
cVal[key] = value
return nil
}
package mxj
// Per: https://github.com/clbanning/mxj/issues/37#issuecomment-278651862
var fieldSep string = ":"
// SetFieldSeparator changes the default field separator, ":", for the
// newVal argument in mv.UpdateValuesForPath and the optional 'subkey' arguments
// in mv.ValuesForKey and mv.ValuesForPath.
//
// E.g., if the newVal value is "http://blah/blah", setting the field separator
// to "|" will allow the newVal specification, "<key>|http://blah/blah" to parse
// properly. If called with no argument or an empty string value, the field
// separator is set to the default, ":".
func SetFieldSeparator(s ...string) {
if len(s) == 0 || s[0] == "" {
fieldSep = ":" // the default
return
}
fieldSep = s[0]
}
// Copyright 2016 Charles Banning. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file
// strict.go actually addresses setting xml.Decoder attribute
// values. This'll let you parse non-standard XML.
package mxj
import (
"encoding/xml"
)
// CustomDecoder can be used to specify xml.Decoder attribute
// values, e.g., Strict:false, to be used. By default CustomDecoder
// is nil. If CustomeDecoder != nil, then mxj.XmlCharsetReader variable is
// ignored and must be set as part of the CustomDecoder value, if needed.
// Usage:
// mxj.CustomDecoder = &xml.Decoder{Strict:false}
var CustomDecoder *xml.Decoder
// useCustomDecoder copy over public attributes from customDecoder
func useCustomDecoder(d *xml.Decoder) {
d.Strict = CustomDecoder.Strict
d.AutoClose = CustomDecoder.AutoClose
d.Entity = CustomDecoder.Entity
d.CharsetReader = CustomDecoder.CharsetReader
d.DefaultSpace = CustomDecoder.DefaultSpace
}
// Copyright 2012-2017 Charles Banning. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file
package mxj
import (
"encoding/json"
"errors"
"reflect"
// "github.com/fatih/structs"
)
// Create a new Map value from a structure. Error returned if argument is not a structure.
// Only public structure fields are decoded in the Map value. See github.com/fatih/structs#Map
// for handling of "structs" tags.
// DEPRECATED - import github.com/fatih/structs and cast result of structs.Map to mxj.Map.
// import "github.com/fatih/structs"
// ...
// sm, err := structs.Map(<some struct>)
// if err != nil {
// // handle error
// }
// m := mxj.Map(sm)
// Alernatively uncomment the old source and import in struct.go.
func NewMapStruct(structVal interface{}) (Map, error) {
return nil, errors.New("deprecated - see package documentation")
/*
if !structs.IsStruct(structVal) {
return nil, errors.New("NewMapStruct() error: argument is not type Struct")
}
return structs.Map(structVal), nil
*/
}
// Marshal a map[string]interface{} into a structure referenced by 'structPtr'. Error returned
// if argument is not a pointer or if json.Unmarshal returns an error.
// json.Unmarshal structure encoding rules are followed to encode public structure fields.
func (mv Map) Struct(structPtr interface{}) error {
// should check that we're getting a pointer.
if reflect.ValueOf(structPtr).Kind() != reflect.Ptr {
return errors.New("mv.Struct() error: argument is not type Ptr")
}
m := map[string]interface{}(mv)
j, err := json.Marshal(m)
if err != nil {
return err
}
return json.Unmarshal(j, structPtr)
}
// Copyright 2012-2014, 2017 Charles Banning. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file
// updatevalues.go - modify a value based on path and possibly sub-keys
// TODO(clb): handle simple elements with attributes and NewMapXmlSeq Map values.
package mxj
import (
"fmt"
"strconv"
"strings"
)
// Update value based on path and possible sub-key values.
// A count of the number of values changed and any error are returned.
// If the count == 0, then no path (and subkeys) matched.
// 'newVal' can be a Map or map[string]interface{} value with a single 'key' that is the key to be modified
// or a string value "key:value[:type]" where type is "bool" or "num" to cast the value.
// 'path' is dot-notation list of keys to traverse; last key in path can be newVal key
// NOTE: 'path' spec does not currently support indexed array references.
// 'subkeys' are "key:value[:type]" entries that must match for path node
// - For attributes prefix the label with the attribute prefix character, by default a
// hyphen, '-', e.g., "-seq:3". (See SetAttrPrefix function.)
// - The subkey can be wildcarded - "key:*" - to require that it's there with some value.
// - If a subkey is preceeded with the '!' character, the key:value[:type] entry is treated as an
// exclusion critera - e.g., "!author:William T. Gaddis".
//
// NOTES:
// 1. Simple elements with attributes need a path terminated as ".#text" to modify the actual value.
// 2. Values in Maps created using NewMapXmlSeq are map[string]interface{} values with a "#text" key.
// 3. If values in 'newVal' or 'subkeys' args contain ":", use SetFieldSeparator to an unused symbol,
// perhaps "|".
func (mv Map) UpdateValuesForPath(newVal interface{}, path string, subkeys ...string) (int, error) {
m := map[string]interface{}(mv)
// extract the subkeys
var subKeyMap map[string]interface{}
if len(subkeys) > 0 {
var err error
subKeyMap, err = getSubKeyMap(subkeys...)
if err != nil {
return 0, err
}
}
// extract key and value from newVal
var key string
var val interface{}
switch newVal.(type) {
case map[string]interface{}, Map:
switch newVal.(type) { // "fallthrough is not permitted in type switch" (Spec)
case Map:
newVal = newVal.(Map).Old()
}
if len(newVal.(map[string]interface{})) != 1 {
return 0, fmt.Errorf("newVal map can only have len == 1 - %+v", newVal)
}
for key, val = range newVal.(map[string]interface{}) {
}
case string: // split it as a key:value pair
ss := strings.Split(newVal.(string), fieldSep)
n := len(ss)
if n < 2 || n > 3 {
return 0, fmt.Errorf("unknown newVal spec - %+v", newVal)
}
key = ss[0]
if n == 2 {
val = interface{}(ss[1])
} else if n == 3 {
switch ss[2] {
case "bool", "boolean":
nv, err := strconv.ParseBool(ss[1])
if err != nil {
return 0, fmt.Errorf("can't convert newVal to bool - %+v", newVal)
}
val = interface{}(nv)
case "num", "numeric", "float", "int":
nv, err := strconv.ParseFloat(ss[1], 64)
if err != nil {
return 0, fmt.Errorf("can't convert newVal to float64 - %+v", newVal)
}
val = interface{}(nv)
default:
return 0, fmt.Errorf("unknown type for newVal value - %+v", newVal)
}
}
default:
return 0, fmt.Errorf("invalid newVal type - %+v", newVal)
}
// parse path
keys := strings.Split(path, ".")
var count int
updateValuesForKeyPath(key, val, m, keys, subKeyMap, &count)
return count, nil
}
// navigate the path
func updateValuesForKeyPath(key string, value interface{}, m interface{}, keys []string, subkeys map[string]interface{}, cnt *int) {
// ----- at end node: looking at possible node to get 'key' ----
if len(keys) == 1 {
updateValue(key, value, m, keys[0], subkeys, cnt)
return
}
// ----- here we are navigating the path thru the penultimate node --------
// key of interest is keys[0] - the next in the path
switch keys[0] {
case "*": // wildcard - scan all values
switch m.(type) {
case map[string]interface{}:
for _, v := range m.(map[string]interface{}) {
updateValuesForKeyPath(key, value, v, keys[1:], subkeys, cnt)
}
case []interface{}:
for _, v := range m.([]interface{}) {
switch v.(type) {
// flatten out a list of maps - keys are processed
case map[string]interface{}:
for _, vv := range v.(map[string]interface{}) {
updateValuesForKeyPath(key, value, vv, keys[1:], subkeys, cnt)
}
default:
updateValuesForKeyPath(key, value, v, keys[1:], subkeys, cnt)
}
}
}
default: // key - must be map[string]interface{}
switch m.(type) {
case map[string]interface{}:
if v, ok := m.(map[string]interface{})[keys[0]]; ok {
updateValuesForKeyPath(key, value, v, keys[1:], subkeys, cnt)
}
case []interface{}: // may be buried in list
for _, v := range m.([]interface{}) {
switch v.(type) {
case map[string]interface{}:
if vv, ok := v.(map[string]interface{})[keys[0]]; ok {
updateValuesForKeyPath(key, value, vv, keys[1:], subkeys, cnt)
}
}
}
}
}
}
// change value if key and subkeys are present
func updateValue(key string, value interface{}, m interface{}, keys0 string, subkeys map[string]interface{}, cnt *int) {
// there are two possible options for the value of 'keys0': map[string]interface, []interface{}
// and 'key' is a key in the map or is a key in a map in a list.
switch m.(type) {
case map[string]interface{}: // gotta have the last key
if keys0 == "*" {
for k := range m.(map[string]interface{}) {
updateValue(key, value, m, k, subkeys, cnt)
}
return
}
endVal, _ := m.(map[string]interface{})[keys0]
// if newV key is the end of path, replace the value for path-end
// may be []interface{} - means replace just an entry w/ subkeys
// otherwise replace the keys0 value if subkeys are there
// NOTE: this will replace the subkeys, also
if key == keys0 {
switch endVal.(type) {
case map[string]interface{}:
if hasSubKeys(m, subkeys) {
(m.(map[string]interface{}))[keys0] = value
(*cnt)++
}
case []interface{}:
// without subkeys can't select list member to modify
// so key:value spec is it ...
if hasSubKeys(m, subkeys) {
(m.(map[string]interface{}))[keys0] = value
(*cnt)++
break
}
nv := make([]interface{}, 0)
var valmodified bool
for _, v := range endVal.([]interface{}) {
// check entry subkeys
if hasSubKeys(v, subkeys) {
// replace v with value
nv = append(nv, value)
valmodified = true
(*cnt)++
continue
}
nv = append(nv, v)
}
if valmodified {
(m.(map[string]interface{}))[keys0] = interface{}(nv)
}
default: // anything else is a strict replacement
if hasSubKeys(m, subkeys) {
(m.(map[string]interface{}))[keys0] = value
(*cnt)++
}
}
return
}
// so value is for an element of endVal
// if endVal is a map then 'key' must be there w/ subkeys
// if endVal is a list then 'key' must be in a list member w/ subkeys
switch endVal.(type) {
case map[string]interface{}:
if !hasSubKeys(endVal, subkeys) {
return
}
if _, ok := (endVal.(map[string]interface{}))[key]; ok {
(endVal.(map[string]interface{}))[key] = value
(*cnt)++
}
case []interface{}: // keys0 points to a list, check subkeys
for _, v := range endVal.([]interface{}) {
// got to be a map so we can replace value for 'key'
vv, vok := v.(map[string]interface{})
if !vok {
continue
}
if _, ok := vv[key]; !ok {
continue
}
if !hasSubKeys(vv, subkeys) {
continue
}
vv[key] = value
(*cnt)++
}
}
case []interface{}: // key may be in a list member
// don't need to handle keys0 == "*"; we're looking at everything, anyway.
for _, v := range m.([]interface{}) {
// only map values - we're looking for 'key'
mm, ok := v.(map[string]interface{})
if !ok {
continue
}
if _, ok := mm[key]; !ok {
continue
}
if !hasSubKeys(mm, subkeys) {
continue
}
mm[key] = value
(*cnt)++
}
}
// return
}
// Copyright 2012-2016, 2018-2019 Charles Banning. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file
// xml.go - basically the core of X2j for map[string]interface{} values.
// NewMapXml, NewMapXmlReader, mv.Xml, mv.XmlWriter
// see x2j and j2x for wrappers to provide end-to-end transformation of XML and JSON messages.
package mxj
import (
"bytes"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"io"
"reflect"
"sort"
"strconv"
"strings"
"time"
)
var (
textK = "#text"
seqK = "#seq"
commentK = "#comment"
attrK = "#attr"
directiveK = "#directive"
procinstK = "#procinst"
targetK = "#target"
instK = "#inst"
)
// Support overriding default Map keys prefix
func SetGlobalKeyMapPrefix(s string) {
textK = strings.ReplaceAll(textK, textK[0:1], s)
seqK = strings.ReplaceAll(seqK, seqK[0:1], s)
commentK = strings.ReplaceAll(commentK, commentK[0:1], s)
directiveK = strings.ReplaceAll(directiveK, directiveK[0:1], s)
procinstK = strings.ReplaceAll(procinstK, procinstK[0:1], s)
targetK = strings.ReplaceAll(targetK, targetK[0:1], s)
instK = strings.ReplaceAll(instK, instK[0:1], s)
attrK = strings.ReplaceAll(attrK, attrK[0:1], s)
}
// ------------------- NewMapXml & NewMapXmlReader ... -------------------------
// If XmlCharsetReader != nil, it will be used to decode the XML, if required.
// Note: if CustomDecoder != nil, then XmlCharsetReader is ignored;
// set the CustomDecoder attribute instead.
// import (
// charset "code.google.com/p/go-charset/charset"
// github.com/clbanning/mxj
// )
// ...
// mxj.XmlCharsetReader = charset.NewReader
// m, merr := mxj.NewMapXml(xmlValue)
var XmlCharsetReader func(charset string, input io.Reader) (io.Reader, error)
// NewMapXml - convert a XML doc into a Map
// (This is analogous to unmarshalling a JSON string to map[string]interface{} using json.Unmarshal().)
// If the optional argument 'cast' is 'true', then values will be converted to boolean or float64 if possible.
//
// Converting XML to JSON is a simple as:
// ...
// mapVal, merr := mxj.NewMapXml(xmlVal)
// if merr != nil {
// // handle error
// }
// jsonVal, jerr := mapVal.Json()
// if jerr != nil {
// // handle error
// }
//
// NOTES:
// 1. Declarations, directives, process instructions and comments are NOT parsed.
// 2. The 'xmlVal' will be parsed looking for an xml.StartElement, so BOM and other
// extraneous xml.CharData will be ignored unless io.EOF is reached first.
// 3. If CoerceKeysToLower() has been called, then all key values will be lower case.
// 4. If CoerceKeysToSnakeCase() has been called, then all key values will be converted to snake case.
// 5. If DisableTrimWhiteSpace(b bool) has been called, then all values will be trimmed or not. 'true' by default.
func NewMapXml(xmlVal []byte, cast ...bool) (Map, error) {
var r bool
if len(cast) == 1 {
r = cast[0]
}
return xmlToMap(xmlVal, r)
}
// Get next XML doc from an io.Reader as a Map value. Returns Map value.
// NOTES:
// 1. Declarations, directives, process instructions and comments are NOT parsed.
// 2. The 'xmlReader' will be parsed looking for an xml.StartElement, so BOM and other
// extraneous xml.CharData will be ignored unless io.EOF is reached first.
// 3. If CoerceKeysToLower() has been called, then all key values will be lower case.
// 4. If CoerceKeysToSnakeCase() has been called, then all key values will be converted to snake case.
func NewMapXmlReader(xmlReader io.Reader, cast ...bool) (Map, error) {
var r bool
if len(cast) == 1 {
r = cast[0]
}
// We need to put an *os.File reader in a ByteReader or the xml.NewDecoder
// will wrap it in a bufio.Reader and seek on the file beyond where the
// xml.Decoder parses!
if _, ok := xmlReader.(io.ByteReader); !ok {
xmlReader = myByteReader(xmlReader) // see code at EOF
}
// build the map
return xmlReaderToMap(xmlReader, r)
}
// Get next XML doc from an io.Reader as a Map value. Returns Map value and slice with the raw XML.
// NOTES:
// 1. Declarations, directives, process instructions and comments are NOT parsed.
// 2. Due to the implementation of xml.Decoder, the raw XML off the reader is buffered to []byte
// using a ByteReader. If the io.Reader is an os.File, there may be significant performance impact.
// See the examples - getmetrics1.go through getmetrics4.go - for comparative use cases on a large
// data set. If the io.Reader is wrapping a []byte value in-memory, however, such as http.Request.Body
// you CAN use it to efficiently unmarshal a XML doc and retrieve the raw XML in a single call.
// 3. The 'raw' return value may be larger than the XML text value.
// 4. The 'xmlReader' will be parsed looking for an xml.StartElement, so BOM and other
// extraneous xml.CharData will be ignored unless io.EOF is reached first.
// 5. If CoerceKeysToLower() has been called, then all key values will be lower case.
// 6. If CoerceKeysToSnakeCase() has been called, then all key values will be converted to snake case.
func NewMapXmlReaderRaw(xmlReader io.Reader, cast ...bool) (Map, []byte, error) {
var r bool
if len(cast) == 1 {
r = cast[0]
}
// create TeeReader so we can retrieve raw XML
buf := make([]byte, 0)
wb := bytes.NewBuffer(buf)
trdr := myTeeReader(xmlReader, wb) // see code at EOF
m, err := xmlReaderToMap(trdr, r)
// retrieve the raw XML that was decoded
b := wb.Bytes()
if err != nil {
return nil, b, err
}
return m, b, nil
}
// xmlReaderToMap() - parse a XML io.Reader to a map[string]interface{} value
func xmlReaderToMap(rdr io.Reader, r bool) (map[string]interface{}, error) {
// parse the Reader
p := xml.NewDecoder(rdr)
if CustomDecoder != nil {
useCustomDecoder(p)
} else {
p.CharsetReader = XmlCharsetReader
}
return xmlToMapParser("", nil, p, r)
}
// xmlToMap - convert a XML doc into map[string]interface{} value
func xmlToMap(doc []byte, r bool) (map[string]interface{}, error) {
b := bytes.NewReader(doc)
p := xml.NewDecoder(b)
if CustomDecoder != nil {
useCustomDecoder(p)
} else {
p.CharsetReader = XmlCharsetReader
}
return xmlToMapParser("", nil, p, r)
}
// ===================================== where the work happens =============================
// PrependAttrWithHyphen. Prepend attribute tags with a hyphen.
// Default is 'true'. (Not applicable to NewMapXmlSeq(), mv.XmlSeq(), etc.)
// Note:
// If 'false', unmarshaling and marshaling is not symmetric. Attributes will be
// marshal'd as <attr_tag>attr</attr_tag> and may be part of a list.
func PrependAttrWithHyphen(v bool) {
if v {
attrPrefix = "-"
lenAttrPrefix = len(attrPrefix)
return
}
attrPrefix = ""
lenAttrPrefix = len(attrPrefix)
}
// Include sequence id with inner tags. - per Sean Murphy, murphysean84@gmail.com.
var includeTagSeqNum bool
// IncludeTagSeqNum - include a "_seq":N key:value pair with each inner tag, denoting
// its position when parsed. This is of limited usefulness, since list values cannot
// be tagged with "_seq" without changing their depth in the Map.
// So THIS SHOULD BE USED WITH CAUTION - see the test cases. Here's a sample of what
// you get.
/*
<Obj c="la" x="dee" h="da">
<IntObj id="3"/>
<IntObj1 id="1"/>
<IntObj id="2"/>
<StrObj>hello</StrObj>
</Obj>
parses as:
{
Obj:{
"-c":"la",
"-h":"da",
"-x":"dee",
"intObj":[
{
"-id"="3",
"_seq":"0" // if mxj.Cast is passed, then: "_seq":0
},
{
"-id"="2",
"_seq":"2"
}],
"intObj1":{
"-id":"1",
"_seq":"1"
},
"StrObj":{
"#text":"hello", // simple element value gets "#text" tag
"_seq":"3"
}
}
}
*/
func IncludeTagSeqNum(b ...bool) {
if len(b) == 0 {
includeTagSeqNum = !includeTagSeqNum
} else if len(b) == 1 {
includeTagSeqNum = b[0]
}
}
// all keys will be "lower case"
var lowerCase bool
// Coerce all tag values to keys in lower case. This is useful if you've got sources with variable
// tag capitalization, and you want to use m.ValuesForKeys(), etc., with the key or path spec
// in lower case.
// CoerceKeysToLower() will toggle the coercion flag true|false - on|off
// CoerceKeysToLower(true|false) will set the coercion flag on|off
//
// NOTE: only recognized by NewMapXml, NewMapXmlReader, and NewMapXmlReaderRaw functions as well as
// the associated HandleXmlReader and HandleXmlReaderRaw.
func CoerceKeysToLower(b ...bool) {
if len(b) == 0 {
lowerCase = !lowerCase
} else if len(b) == 1 {
lowerCase = b[0]
}
}
// disableTrimWhiteSpace sets if the white space should be removed or not
var disableTrimWhiteSpace bool
var trimRunes = "\t\r\b\n "
// DisableTrimWhiteSpace set if the white space should be trimmed or not. By default white space is always trimmed. If
// no argument is provided, trim white space will be disabled.
func DisableTrimWhiteSpace(b ...bool) {
if len(b) == 0 {
disableTrimWhiteSpace = true
} else {
disableTrimWhiteSpace = b[0]
}
if disableTrimWhiteSpace {
trimRunes = "\t\r\b\n"
} else {
trimRunes = "\t\r\b\n "
}
}
// 25jun16: Allow user to specify the "prefix" character for XML attribute key labels.
// We do this by replacing '`' constant with attrPrefix var, replacing useHyphen with attrPrefix = "",
// and adding a SetAttrPrefix(s string) function.
var attrPrefix string = `-` // the default
var lenAttrPrefix int = 1 // the default
// SetAttrPrefix changes the default, "-", to the specified value, s.
// SetAttrPrefix("") is the same as PrependAttrWithHyphen(false).
// (Not applicable for NewMapXmlSeq(), mv.XmlSeq(), etc.)
func SetAttrPrefix(s string) {
attrPrefix = s
lenAttrPrefix = len(attrPrefix)
}
// 18jan17: Allows user to specify if the map keys should be in snake case instead
// of the default hyphenated notation.
var snakeCaseKeys bool
// CoerceKeysToSnakeCase changes the default, false, to the specified value, b.
// Note: the attribute prefix will be a hyphen, '-', or what ever string value has
// been specified using SetAttrPrefix.
func CoerceKeysToSnakeCase(b ...bool) {
if len(b) == 0 {
snakeCaseKeys = !snakeCaseKeys
} else if len(b) == 1 {
snakeCaseKeys = b[0]
}
}
// 10jan19: use of pull request #57 should be conditional - legacy code assumes
// numeric values are float64.
var castToInt bool
// CastValuesToInt tries to coerce numeric valus to int64 or uint64 instead of the
// default float64. Repeated calls with no argument will toggle this on/off, or this
// handling will be set with the value of 'b'.
func CastValuesToInt(b ...bool) {
if len(b) == 0 {
castToInt = !castToInt
} else if len(b) == 1 {
castToInt = b[0]
}
}
// 05feb17: support processing XMPP streams (issue #36)
var handleXMPPStreamTag bool
// HandleXMPPStreamTag causes decoder to parse XMPP <stream:stream> elements.
// If called with no argument, XMPP stream element handling is toggled on/off.
// (See xmppStream_test.go for example.)
// If called with NewMapXml, NewMapXmlReader, New MapXmlReaderRaw the "stream"
// element will be returned as:
// map["stream"]interface{}{map[-<attrs>]interface{}}.
// If called with NewMapSeq, NewMapSeqReader, NewMapSeqReaderRaw the "stream"
// element will be returned as:
// map["stream:stream"]interface{}{map["#attr"]interface{}{map[string]interface{}}}
// where the "#attr" values have "#text" and "#seq" keys. (See NewMapXmlSeq.)
func HandleXMPPStreamTag(b ...bool) {
if len(b) == 0 {
handleXMPPStreamTag = !handleXMPPStreamTag
} else if len(b) == 1 {
handleXMPPStreamTag = b[0]
}
}
// 21jan18 - decode all values as map["#text":value] (issue #56)
var decodeSimpleValuesAsMap bool
// DecodeSimpleValuesAsMap forces all values to be decoded as map["#text":<value>].
// If called with no argument, the decoding is toggled on/off.
//
// By default the NewMapXml functions decode simple values without attributes as
// map[<tag>:<value>]. This function causes simple values without attributes to be
// decoded the same as simple values with attributes - map[<tag>:map["#text":<value>]].
func DecodeSimpleValuesAsMap(b ...bool) {
if len(b) == 0 {
decodeSimpleValuesAsMap = !decodeSimpleValuesAsMap
} else if len(b) == 1 {
decodeSimpleValuesAsMap = b[0]
}
}
// xmlToMapParser (2015.11.12) - load a 'clean' XML doc into a map[string]interface{} directly.
// A refactoring of xmlToTreeParser(), markDuplicate() and treeToMap() - here, all-in-one.
// We've removed the intermediate *node tree with the allocation and subsequent rescanning.
func xmlToMapParser(skey string, a []xml.Attr, p *xml.Decoder, r bool) (map[string]interface{}, error) {
if lowerCase {
skey = strings.ToLower(skey)
}
if snakeCaseKeys {
skey = strings.Replace(skey, "-", "_", -1)
}
// NOTE: all attributes and sub-elements parsed into 'na', 'na' is returned as value for 'skey' in 'n'.
// Unless 'skey' is a simple element w/o attributes, in which case the xml.CharData value is the value.
var n, na map[string]interface{}
var seq int // for includeTagSeqNum
// Allocate maps and load attributes, if any.
// NOTE: on entry from NewMapXml(), etc., skey=="", and we fall through
// to get StartElement then recurse with skey==xml.StartElement.Name.Local
// where we begin allocating map[string]interface{} values 'n' and 'na'.
if skey != "" {
n = make(map[string]interface{}) // old n
na = make(map[string]interface{}) // old n.nodes
if len(a) > 0 {
for _, v := range a {
if snakeCaseKeys {
v.Name.Local = strings.Replace(v.Name.Local, "-", "_", -1)
}
var key string
key = attrPrefix + v.Name.Local
if lowerCase {
key = strings.ToLower(key)
}
if xmlEscapeCharsDecoder { // per issue#84
v.Value = escapeChars(v.Value)
}
na[key] = cast(v.Value, r, key)
}
}
}
// Return XMPP <stream:stream> message.
if handleXMPPStreamTag && skey == "stream" {
n[skey] = na
return n, nil
}
for {
t, err := p.Token()
if err != nil {
if err != io.EOF {
return nil, errors.New("xml.Decoder.Token() - " + err.Error())
}
return nil, err
}
switch t.(type) {
case xml.StartElement:
tt := t.(xml.StartElement)
// First call to xmlToMapParser() doesn't pass xml.StartElement - the map key.
// So when the loop is first entered, the first token is the root tag along
// with any attributes, which we process here.
//
// Subsequent calls to xmlToMapParser() will pass in tag+attributes for
// processing before getting the next token which is the element value,
// which is done above.
if skey == "" {
return xmlToMapParser(tt.Name.Local, tt.Attr, p, r)
}
// If not initializing the map, parse the element.
// len(nn) == 1, necessarily - it is just an 'n'.
nn, err := xmlToMapParser(tt.Name.Local, tt.Attr, p, r)
if err != nil {
return nil, err
}
// The nn map[string]interface{} value is a na[nn_key] value.
// We need to see if nn_key already exists - means we're parsing a list.
// This may require converting na[nn_key] value into []interface{} type.
// First, extract the key:val for the map - it's a singleton.
// Note:
// * if CoerceKeysToLower() called, then key will be lower case.
// * if CoerceKeysToSnakeCase() called, then key will be converted to snake case.
var key string
var val interface{}
for key, val = range nn {
break
}
// IncludeTagSeqNum requests that the element be augmented with a "_seq" sub-element.
// In theory, we don't need this if len(na) == 1. But, we don't know what might
// come next - we're only parsing forward. So if you ask for 'includeTagSeqNum' you
// get it on every element. (Personally, I never liked this, but I added it on request
// and did get a $50 Amazon gift card in return - now we support it for backwards compatibility!)
if includeTagSeqNum {
switch val.(type) {
case []interface{}:
// noop - There's no clean way to handle this w/o changing message structure.
case map[string]interface{}:
val.(map[string]interface{})["_seq"] = seq // will overwrite an "_seq" XML tag
seq++
case interface{}: // a non-nil simple element: string, float64, bool
v := map[string]interface{}{textK: val}
v["_seq"] = seq
seq++
val = v
}
}
// 'na' holding sub-elements of n.
// See if 'key' already exists.
// If 'key' exists, then this is a list, if not just add key:val to na.
if v, ok := na[key]; ok {
var a []interface{}
switch v.(type) {
case []interface{}:
a = v.([]interface{})
default: // anything else - note: v.(type) != nil
a = []interface{}{v}
}
a = append(a, val)
na[key] = a
} else {
na[key] = val // save it as a singleton
}
case xml.EndElement:
// len(n) > 0 if this is a simple element w/o xml.Attrs - see xml.CharData case.
if len(n) == 0 {
// If len(na)==0 we have an empty element == "";
// it has no xml.Attr nor xml.CharData.
// Note: in original node-tree parser, val defaulted to "";
// so we always had the default if len(node.nodes) == 0.
if len(na) > 0 {
n[skey] = na
} else {
n[skey] = "" // empty element
}
} else if len(n) == 1 && len(na) > 0 {
// it's a simple element w/ no attributes w/ subelements
for _, v := range n {
na[textK] = v
}
n[skey] = na
}
return n, nil
case xml.CharData:
// clean up possible noise
tt := strings.Trim(string(t.(xml.CharData)), trimRunes)
if xmlEscapeCharsDecoder { // issue#84
tt = escapeChars(tt)
}
if len(tt) > 0 {
if len(na) > 0 || decodeSimpleValuesAsMap {
na[textK] = cast(tt, r, textK)
} else if skey != "" {
n[skey] = cast(tt, r, skey)
} else {
// per Adrian (http://www.adrianlungu.com/) catch stray text
// in decoder stream -
// https://github.com/clbanning/mxj/pull/14#issuecomment-182816374
// NOTE: CharSetReader must be set to non-UTF-8 CharSet or you'll get
// a p.Token() decoding error when the BOM is UTF-16 or UTF-32.
continue
}
}
default:
// noop
}
}
}
var castNanInf bool
// Cast "Nan", "Inf", "-Inf" XML values to 'float64'.
// By default, these values will be decoded as 'string'.
func CastNanInf(b ...bool) {
if len(b) == 0 {
castNanInf = !castNanInf
} else if len(b) == 1 {
castNanInf = b[0]
}
}
// cast - try to cast string values to bool or float64
// 't' is the tag key that can be checked for 'not-casting'
func cast(s string, r bool, t string) interface{} {
if checkTagToSkip != nil && t != "" && checkTagToSkip(t) {
// call the check-function here with 't[0]'
// if 'true' return s
return s
}
if r {
// handle nan and inf
if !castNanInf {
switch strings.ToLower(s) {
case "nan", "inf", "-inf":
return s
}
}
// handle numeric strings ahead of boolean
if castToInt {
if f, err := strconv.ParseInt(s, 10, 64); err == nil {
return f
}
if f, err := strconv.ParseUint(s, 10, 64); err == nil {
return f
}
}
if castToFloat {
if f, err := strconv.ParseFloat(s, 64); err == nil {
return f
}
}
// ParseBool treats "1"==true & "0"==false, we've already scanned those
// values as float64. See if value has 't' or 'f' as initial screen to
// minimize calls to ParseBool; also, see if len(s) < 6.
if castToBool {
if len(s) > 0 && len(s) < 6 {
switch s[:1] {
case "t", "T", "f", "F":
if b, err := strconv.ParseBool(s); err == nil {
return b
}
}
}
}
}
return s
}
// pull request, #59
var castToFloat = true
// CastValuesToFloat can be used to skip casting to float64 when
// "cast" argument is 'true' in NewMapXml, etc.
// Default is true.
func CastValuesToFloat(b ...bool) {
if len(b) == 0 {
castToFloat = !castToFloat
} else if len(b) == 1 {
castToFloat = b[0]
}
}
var castToBool = true
// CastValuesToBool can be used to skip casting to bool when
// "cast" argument is 'true' in NewMapXml, etc.
// Default is true.
func CastValuesToBool(b ...bool) {
if len(b) == 0 {
castToBool = !castToBool
} else if len(b) == 1 {
castToBool = b[0]
}
}
// checkTagToSkip - switch to address Issue #58
var checkTagToSkip func(string) bool
// SetCheckTagToSkipFunc registers function to test whether the value
// for a tag should be cast to bool or float64 when "cast" argument is 'true'.
// (Dot tag path notation is not supported.)
// NOTE: key may be "#text" if it's a simple element with attributes
// or "decodeSimpleValuesAsMap == true".
// NOTE: does not apply to NewMapXmlSeq... functions.
func SetCheckTagToSkipFunc(fn func(string) bool) {
checkTagToSkip = fn
}
// ------------------ END: NewMapXml & NewMapXmlReader -------------------------
// ------------------ mv.Xml & mv.XmlWriter - from j2x ------------------------
const (
DefaultRootTag = "doc"
)
var useGoXmlEmptyElemSyntax bool
// XmlGoEmptyElemSyntax() - <tag ...></tag> rather than <tag .../>.
// Go's encoding/xml package marshals empty XML elements as <tag ...></tag>. By default this package
// encodes empty elements as <tag .../>. If you're marshaling Map values that include structures
// (which are passed to xml.Marshal for encoding), this will let you conform to the standard package.
func XmlGoEmptyElemSyntax() {
useGoXmlEmptyElemSyntax = true
}
// XmlDefaultEmptyElemSyntax() - <tag .../> rather than <tag ...></tag>.
// Return XML encoding for empty elements to the default package setting.
// Reverses effect of XmlGoEmptyElemSyntax().
func XmlDefaultEmptyElemSyntax() {
useGoXmlEmptyElemSyntax = false
}
// ------- issue #88 ----------
// xmlCheckIsValid set switch to force decoding the encoded XML to
// see if it is valid XML.
var xmlCheckIsValid bool
// XmlCheckIsValid forces the encoded XML to be checked for validity.
func XmlCheckIsValid(b ...bool) {
if len(b) == 1 {
xmlCheckIsValid = b[0]
return
}
xmlCheckIsValid = !xmlCheckIsValid
}
// Encode a Map as XML. The companion of NewMapXml().
// The following rules apply.
// - The key label "#text" is treated as the value for a simple element with attributes.
// - Map keys that begin with a hyphen, '-', are interpreted as attributes.
// It is an error if the attribute doesn't have a []byte, string, number, or boolean value.
// - Map value type encoding:
// > string, bool, float64, int, int32, int64, float32: per "%v" formating
// > []bool, []uint8: by casting to string
// > structures, etc.: handed to xml.Marshal() - if there is an error, the element
// value is "UNKNOWN"
// - Elements with only attribute values or are null are terminated using "/>".
// - If len(mv) == 1 and no rootTag is provided, then the map key is used as the root tag, possible.
// Thus, `{ "key":"value" }` encodes as "<key>value</key>".
// - To encode empty elements in a syntax consistent with encoding/xml call UseGoXmlEmptyElementSyntax().
// The attributes tag=value pairs are alphabetized by "tag". Also, when encoding map[string]interface{} values -
// complex elements, etc. - the key:value pairs are alphabetized by key so the resulting tags will appear sorted.
func (mv Map) Xml(rootTag ...string) ([]byte, error) {
m := map[string]interface{}(mv)
var err error
b := new(bytes.Buffer)
p := new(pretty) // just a stub
if len(m) == 1 && len(rootTag) == 0 {
for key, value := range m {
// if it an array, see if all values are map[string]interface{}
// we force a new root tag if we'll end up with no key:value in the list
// so: key:[string_val, bool:true] --> <doc><key>string_val</key><bool>true</bool></doc>
switch value.(type) {
case []interface{}:
for _, v := range value.([]interface{}) {
switch v.(type) {
case map[string]interface{}: // noop
default: // anything else
err = marshalMapToXmlIndent(false, b, DefaultRootTag, m, p)
goto done
}
}
}
err = marshalMapToXmlIndent(false, b, key, value, p)
}
} else if len(rootTag) == 1 {
err = marshalMapToXmlIndent(false, b, rootTag[0], m, p)
} else {
err = marshalMapToXmlIndent(false, b, DefaultRootTag, m, p)
}
done:
if xmlCheckIsValid {
d := xml.NewDecoder(bytes.NewReader(b.Bytes()))
for {
_, err = d.Token()
if err == io.EOF {
err = nil
break
} else if err != nil {
return nil, err
}
}
}
return b.Bytes(), err
}
// The following implementation is provided only for symmetry with NewMapXmlReader[Raw]
// The names will also provide a key for the number of return arguments.
// Writes the Map as XML on the Writer.
// See Xml() for encoding rules.
func (mv Map) XmlWriter(xmlWriter io.Writer, rootTag ...string) error {
x, err := mv.Xml(rootTag...)
if err != nil {
return err
}
_, err = xmlWriter.Write(x)
return err
}
// Writes the Map as XML on the Writer. []byte is the raw XML that was written.
// See Xml() for encoding rules.
/*
func (mv Map) XmlWriterRaw(xmlWriter io.Writer, rootTag ...string) ([]byte, error) {
x, err := mv.Xml(rootTag...)
if err != nil {
return x, err
}
_, err = xmlWriter.Write(x)
return x, err
}
*/
// Writes the Map as pretty XML on the Writer.
// See Xml() for encoding rules.
func (mv Map) XmlIndentWriter(xmlWriter io.Writer, prefix, indent string, rootTag ...string) error {
x, err := mv.XmlIndent(prefix, indent, rootTag...)
if err != nil {
return err
}
_, err = xmlWriter.Write(x)
return err
}
// Writes the Map as pretty XML on the Writer. []byte is the raw XML that was written.
// See Xml() for encoding rules.
/*
func (mv Map) XmlIndentWriterRaw(xmlWriter io.Writer, prefix, indent string, rootTag ...string) ([]byte, error) {
x, err := mv.XmlIndent(prefix, indent, rootTag...)
if err != nil {
return x, err
}
_, err = xmlWriter.Write(x)
return x, err
}
*/
// -------------------- END: mv.Xml & mv.XmlWriter -------------------------------
// -------------- Handle XML stream by processing Map value --------------------
// Default poll delay to keep Handler from spinning on an open stream
// like sitting on os.Stdin waiting for imput.
var xhandlerPollInterval = time.Millisecond
// Bulk process XML using handlers that process a Map value.
// 'rdr' is an io.Reader for XML (stream)
// 'mapHandler' is the Map processor. Return of 'false' stops io.Reader processing.
// 'errHandler' is the error processor. Return of 'false' stops io.Reader processing and returns the error.
// Note: mapHandler() and errHandler() calls are blocking, so reading and processing of messages is serialized.
// This means that you can stop reading the file on error or after processing a particular message.
// To have reading and handling run concurrently, pass argument to a go routine in handler and return 'true'.
func HandleXmlReader(xmlReader io.Reader, mapHandler func(Map) bool, errHandler func(error) bool) error {
var n int
for {
m, merr := NewMapXmlReader(xmlReader)
n++
// handle error condition with errhandler
if merr != nil && merr != io.EOF {
merr = fmt.Errorf("[xmlReader: %d] %s", n, merr.Error())
if ok := errHandler(merr); !ok {
// caused reader termination
return merr
}
continue
}
// pass to maphandler
if len(m) != 0 {
if ok := mapHandler(m); !ok {
break
}
} else if merr != io.EOF {
time.Sleep(xhandlerPollInterval)
}
if merr == io.EOF {
break
}
}
return nil
}
// Bulk process XML using handlers that process a Map value and the raw XML.
// 'rdr' is an io.Reader for XML (stream)
// 'mapHandler' is the Map and raw XML - []byte - processor. Return of 'false' stops io.Reader processing.
// 'errHandler' is the error and raw XML processor. Return of 'false' stops io.Reader processing and returns the error.
// Note: mapHandler() and errHandler() calls are blocking, so reading and processing of messages is serialized.
// This means that you can stop reading the file on error or after processing a particular message.
// To have reading and handling run concurrently, pass argument(s) to a go routine in handler and return 'true'.
// See NewMapXmlReaderRaw for comment on performance associated with retrieving raw XML from a Reader.
func HandleXmlReaderRaw(xmlReader io.Reader, mapHandler func(Map, []byte) bool, errHandler func(error, []byte) bool) error {
var n int
for {
m, raw, merr := NewMapXmlReaderRaw(xmlReader)
n++
// handle error condition with errhandler
if merr != nil && merr != io.EOF {
merr = fmt.Errorf("[xmlReader: %d] %s", n, merr.Error())
if ok := errHandler(merr, raw); !ok {
// caused reader termination
return merr
}
continue
}
// pass to maphandler
if len(m) != 0 {
if ok := mapHandler(m, raw); !ok {
break
}
} else if merr != io.EOF {
time.Sleep(xhandlerPollInterval)
}
if merr == io.EOF {
break
}
}
return nil
}
// ----------------- END: Handle XML stream by processing Map value --------------
// -------- a hack of io.TeeReader ... need one that's an io.ByteReader for xml.NewDecoder() ----------
// This is a clone of io.TeeReader with the additional method t.ReadByte().
// Thus, this TeeReader is also an io.ByteReader.
// This is necessary because xml.NewDecoder uses a ByteReader not a Reader. It appears to have been written
// with bufio.Reader or bytes.Reader in mind ... not a generic io.Reader, which doesn't have to have ReadByte()..
// If NewDecoder is passed a Reader that does not satisfy ByteReader() it wraps the Reader with
// bufio.NewReader and uses ReadByte rather than Read that runs the TeeReader pipe logic.
type teeReader struct {
r io.Reader
w io.Writer
b []byte
}
func myTeeReader(r io.Reader, w io.Writer) io.Reader {
b := make([]byte, 1)
return &teeReader{r, w, b}
}
// need for io.Reader - but we don't use it ...
func (t *teeReader) Read(p []byte) (int, error) {
return 0, nil
}
func (t *teeReader) ReadByte() (byte, error) {
n, err := t.r.Read(t.b)
if n > 0 {
if _, err := t.w.Write(t.b[:1]); err != nil {
return t.b[0], err
}
}
return t.b[0], err
}
// For use with NewMapXmlReader & NewMapXmlSeqReader.
type byteReader struct {
r io.Reader
b []byte
}
func myByteReader(r io.Reader) io.Reader {
b := make([]byte, 1)
return &byteReader{r, b}
}
// Need for io.Reader interface ...
// Needed if reading a malformed http.Request.Body - issue #38.
func (b *byteReader) Read(p []byte) (int, error) {
return b.r.Read(p)
}
func (b *byteReader) ReadByte() (byte, error) {
_, err := b.r.Read(b.b)
if len(b.b) > 0 {
// issue #38
return b.b[0], err
}
var c byte
return c, err
}
// ----------------------- END: io.TeeReader hack -----------------------------------
// ---------------------- XmlIndent - from j2x package ----------------------------
// Encode a map[string]interface{} as a pretty XML string.
// See Xml for encoding rules.
func (mv Map) XmlIndent(prefix, indent string, rootTag ...string) ([]byte, error) {
m := map[string]interface{}(mv)
var err error
b := new(bytes.Buffer)
p := new(pretty)
p.indent = indent
p.padding = prefix
if len(m) == 1 && len(rootTag) == 0 {
// this can extract the key for the single map element
// use it if it isn't a key for a list
for key, value := range m {
if _, ok := value.([]interface{}); ok {
err = marshalMapToXmlIndent(true, b, DefaultRootTag, m, p)
} else {
err = marshalMapToXmlIndent(true, b, key, value, p)
}
}
} else if len(rootTag) == 1 {
err = marshalMapToXmlIndent(true, b, rootTag[0], m, p)
} else {
err = marshalMapToXmlIndent(true, b, DefaultRootTag, m, p)
}
if xmlCheckIsValid {
d := xml.NewDecoder(bytes.NewReader(b.Bytes()))
for {
_, err = d.Token()
if err == io.EOF {
err = nil
break
} else if err != nil {
return nil, err
}
}
}
return b.Bytes(), err
}
type pretty struct {
indent string
cnt int
padding string
mapDepth int
start int
}
func (p *pretty) Indent() {
p.padding += p.indent
p.cnt++
}
func (p *pretty) Outdent() {
if p.cnt > 0 {
p.padding = p.padding[:len(p.padding)-len(p.indent)]
p.cnt--
}
}
// where the work actually happens
// returns an error if an attribute is not atomic
// NOTE: 01may20 - replaces mapToXmlIndent(); uses bytes.Buffer instead for string appends.
func marshalMapToXmlIndent(doIndent bool, b *bytes.Buffer, key string, value interface{}, pp *pretty) error {
var err error
var endTag bool
var isSimple bool
var elen int
p := &pretty{pp.indent, pp.cnt, pp.padding, pp.mapDepth, pp.start}
// per issue #48, 18apr18 - try and coerce maps to map[string]interface{}
// Don't need for mapToXmlSeqIndent, since maps there are decoded by NewMapXmlSeq().
if reflect.ValueOf(value).Kind() == reflect.Map {
switch value.(type) {
case map[string]interface{}:
default:
val := make(map[string]interface{})
vv := reflect.ValueOf(value)
keys := vv.MapKeys()
for _, k := range keys {
val[fmt.Sprint(k)] = vv.MapIndex(k).Interface()
}
value = val
}
}
// 14jul20. The following block of code has become something of a catch all for odd stuff
// that might be passed in as a result of casting an arbitrary map[<T>]<T> to an mxj.Map
// value and then call m.Xml or m.XmlIndent. See issue #71 (and #73) for such edge cases.
switch value.(type) {
// these types are handled during encoding
case map[string]interface{}, []byte, string, float64, bool, int, int32, int64, float32, json.Number:
case []map[string]interface{}, []string, []float64, []bool, []int, []int32, []int64, []float32, []json.Number:
case []interface{}:
case nil:
value = ""
default:
// see if value is a struct, if so marshal using encoding/xml package
if reflect.ValueOf(value).Kind() == reflect.Struct {
if v, err := xml.Marshal(value); err != nil {
return err
} else {
value = string(v)
}
} else {
// coerce eveything else into a string value
value = fmt.Sprint(value)
}
}
// start the XML tag with required indentaton and padding
if doIndent {
switch value.(type) {
case []interface{}, []string:
// list processing handles indentation for all elements
default:
if _, err = b.WriteString(p.padding); err != nil {
return err
}
}
}
switch value.(type) {
case []interface{}:
default:
if _, err = b.WriteString(`<` + key); err != nil {
return err
}
}
switch value.(type) {
case map[string]interface{}:
vv := value.(map[string]interface{})
lenvv := len(vv)
// scan out attributes - attribute keys have prepended attrPrefix
attrlist := make([][2]string, len(vv))
var n int
var ss string
for k, v := range vv {
if lenAttrPrefix > 0 && lenAttrPrefix < len(k) && k[:lenAttrPrefix] == attrPrefix {
switch v.(type) {
case string:
if xmlEscapeChars {
ss = escapeChars(v.(string))
} else {
ss = v.(string)
}
attrlist[n][0] = k[lenAttrPrefix:]
attrlist[n][1] = ss
case float64, bool, int, int32, int64, float32, json.Number:
attrlist[n][0] = k[lenAttrPrefix:]
attrlist[n][1] = fmt.Sprintf("%v", v)
case []byte:
if xmlEscapeChars {
ss = escapeChars(string(v.([]byte)))
} else {
ss = string(v.([]byte))
}
attrlist[n][0] = k[lenAttrPrefix:]
attrlist[n][1] = ss
default:
return fmt.Errorf("invalid attribute value for: %s:<%T>", k, v)
}
n++
}
}
if n > 0 {
attrlist = attrlist[:n]
sort.Sort(attrList(attrlist))
for _, v := range attrlist {
if _, err = b.WriteString(` ` + v[0] + `="` + v[1] + `"`); err != nil {
return err
}
}
}
// only attributes?
if n == lenvv {
if useGoXmlEmptyElemSyntax {
if _, err = b.WriteString(`</` + key + ">"); err != nil {
return err
}
} else {
if _, err = b.WriteString(`/>`); err != nil {
return err
}
}
break
}
// simple element? Note: '#text" is an invalid XML tag.
isComplex := false
if v, ok := vv[textK]; ok && n+1 == lenvv {
// just the value and attributes
switch v.(type) {
case string:
if xmlEscapeChars {
v = escapeChars(v.(string))
} else {
v = v.(string)
}
case []byte:
if xmlEscapeChars {
v = escapeChars(string(v.([]byte)))
} else {
v = string(v.([]byte))
}
}
if _, err = b.WriteString(">" + fmt.Sprintf("%v", v)); err != nil {
return err
}
endTag = true
elen = 1
isSimple = true
break
} else if ok {
// need to handle when there are subelements in addition to the simple element value
// issue #90
switch v.(type) {
case string:
if xmlEscapeChars {
v = escapeChars(v.(string))
} else {
v = v.(string)
}
case []byte:
if xmlEscapeChars {
v = escapeChars(string(v.([]byte)))
} else {
v = string(v.([]byte))
}
}
if _, err = b.WriteString(">" + fmt.Sprintf("%v", v)); err != nil {
return err
}
isComplex = true
}
// close tag with possible attributes
if !isComplex {
if _, err = b.WriteString(">"); err != nil {
return err
}
}
if doIndent {
// *s += "\n"
if _, err = b.WriteString("\n"); err != nil {
return err
}
}
// something more complex
p.mapDepth++
// extract the map k:v pairs and sort on key
elemlist := make([][2]interface{}, len(vv))
n = 0
for k, v := range vv {
if k == textK {
// simple element handled above
continue
}
if lenAttrPrefix > 0 && lenAttrPrefix < len(k) && k[:lenAttrPrefix] == attrPrefix {
continue
}
elemlist[n][0] = k
elemlist[n][1] = v
n++
}
elemlist = elemlist[:n]
sort.Sort(elemList(elemlist))
var i int
for _, v := range elemlist {
switch v[1].(type) {
case []interface{}:
default:
if i == 0 && doIndent {
p.Indent()
}
}
i++
if err := marshalMapToXmlIndent(doIndent, b, v[0].(string), v[1], p); err != nil {
return err
}
switch v[1].(type) {
case []interface{}: // handled in []interface{} case
default:
if doIndent {
p.Outdent()
}
}
i--
}
p.mapDepth--
endTag = true
elen = 1 // we do have some content ...
case []interface{}:
// special case - found during implementing Issue #23
if len(value.([]interface{})) == 0 {
if doIndent {
if _, err = b.WriteString(p.padding + p.indent); err != nil {
return err
}
}
if _, err = b.WriteString("<" + key); err != nil {
return err
}
elen = 0
endTag = true
break
}
for _, v := range value.([]interface{}) {
if doIndent {
p.Indent()
}
if err := marshalMapToXmlIndent(doIndent, b, key, v, p); err != nil {
return err
}
if doIndent {
p.Outdent()
}
}
return nil
case []string:
// This was added by https://github.com/slotix ... not a type that
// would be encountered if mv generated from NewMapXml, NewMapJson.
// Could be encountered in AnyXml(), so we'll let it stay, though
// it should be merged with case []interface{}, above.
//quick fix for []string type
//[]string should be treated exaclty as []interface{}
if len(value.([]string)) == 0 {
if doIndent {
if _, err = b.WriteString(p.padding + p.indent); err != nil {
return err
}
}
if _, err = b.WriteString("<" + key); err != nil {
return err
}
elen = 0
endTag = true
break
}
for _, v := range value.([]string) {
if doIndent {
p.Indent()
}
if err := marshalMapToXmlIndent(doIndent, b, key, v, p); err != nil {
return err
}
if doIndent {
p.Outdent()
}
}
return nil
case nil:
// terminate the tag
if doIndent {
// *s += p.padding
if _, err = b.WriteString(p.padding); err != nil {
return err
}
}
if _, err = b.WriteString("<" + key); err != nil {
return err
}
endTag, isSimple = true, true
break
default: // handle anything - even goofy stuff
elen = 0
switch value.(type) {
case string:
v := value.(string)
if xmlEscapeChars {
v = escapeChars(v)
}
elen = len(v)
if elen > 0 {
// *s += ">" + v
if _, err = b.WriteString(">" + v); err != nil {
return err
}
}
case float64, bool, int, int32, int64, float32, json.Number:
v := fmt.Sprintf("%v", value)
elen = len(v) // always > 0
if _, err = b.WriteString(">" + v); err != nil {
return err
}
case []byte: // NOTE: byte is just an alias for uint8
// similar to how xml.Marshal handles []byte structure members
v := string(value.([]byte))
if xmlEscapeChars {
v = escapeChars(v)
}
elen = len(v)
if elen > 0 {
// *s += ">" + v
if _, err = b.WriteString(">" + v); err != nil {
return err
}
}
default:
if _, err = b.WriteString(">"); err != nil {
return err
}
var v []byte
var err error
if doIndent {
v, err = xml.MarshalIndent(value, p.padding, p.indent)
} else {
v, err = xml.Marshal(value)
}
if err != nil {
if _, err = b.WriteString(">UNKNOWN"); err != nil {
return err
}
} else {
elen = len(v)
if elen > 0 {
if _, err = b.Write(v); err != nil {
return err
}
}
}
}
isSimple = true
endTag = true
}
if endTag {
if doIndent {
if !isSimple {
if _, err = b.WriteString(p.padding); err != nil {
return err
}
}
}
if elen > 0 || useGoXmlEmptyElemSyntax {
if elen == 0 {
if _, err = b.WriteString(">"); err != nil {
return err
}
}
if _, err = b.WriteString(`</` + key + ">"); err != nil {
return err
}
} else {
if _, err = b.WriteString(`/>`); err != nil {
return err
}
}
}
if doIndent {
if p.cnt > p.start {
if _, err = b.WriteString("\n"); err != nil {
return err
}
}
p.Outdent()
}
return nil
}
// ============================ sort interface implementation =================
type attrList [][2]string
func (a attrList) Len() int {
return len(a)
}
func (a attrList) Swap(i, j int) {
a[i], a[j] = a[j], a[i]
}
func (a attrList) Less(i, j int) bool {
return a[i][0] <= a[j][0]
}
type elemList [][2]interface{}
func (e elemList) Len() int {
return len(e)
}
func (e elemList) Swap(i, j int) {
e[i], e[j] = e[j], e[i]
}
func (e elemList) Less(i, j int) bool {
return e[i][0].(string) <= e[j][0].(string)
}
// Copyright 2012-2016, 2019 Charles Banning. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file
// xmlseq.go - version of xml.go with sequence # injection on Decoding and sorting on Encoding.
// Also, handles comments, directives and process instructions.
package mxj
import (
"bytes"
"encoding/xml"
"errors"
"fmt"
"io"
"regexp"
"sort"
"strings"
)
// MapSeq is like Map but contains seqencing indices to allow recovering the original order of
// the XML elements when the map[string]interface{} is marshaled. Element attributes are
// stored as a map["#attr"]map[<attr_key>]map[string]interface{}{"#text":"<value>", "#seq":<attr_index>}
// value instead of denoting the keys with a prefix character. Also, comments, directives and
// process instructions are preserved.
type MapSeq map[string]interface{}
// NoRoot is returned by NewXmlSeq, etc., when a comment, directive or procinstr element is parsed
// in the XML data stream and the element is not contained in an XML object with a root element.
var NoRoot = errors.New("no root key")
var NO_ROOT = NoRoot // maintain backwards compatibility
// ------------------- NewMapXmlSeq & NewMapXmlSeqReader ... -------------------------
// NewMapXmlSeq converts a XML doc into a MapSeq value with elements id'd with decoding sequence key represented
// as map["#seq"]<int value>.
// If the optional argument 'cast' is 'true', then values will be converted to boolean or float64 if possible.
// NOTE: "#seq" key/value pairs are removed on encoding with msv.Xml() / msv.XmlIndent().
//
// - attributes are a map - map["#attr"]map["attr_key"]map[string]interface{}{"#text":<aval>, "#seq":<num>}
//
// - all simple elements are decoded as map["#text"]interface{} with a "#seq" k:v pair, as well.
//
// - lists always decode as map["list_tag"][]map[string]interface{} where the array elements are maps that
// include a "#seq" k:v pair based on sequence they are decoded. Thus, XML like:
// <doc>
// <ltag>value 1</ltag>
// <newtag>value 2</newtag>
// <ltag>value 3</ltag>
// </doc>
// is decoded as:
// doc :
// ltag :[[]interface{}]
// [item: 0]
// #seq :[int] 0
// #text :[string] value 1
// [item: 1]
// #seq :[int] 2
// #text :[string] value 3
// newtag :
// #seq :[int] 1
// #text :[string] value 2
// It will encode in proper sequence even though the MapSeq representation merges all "ltag" elements in an array.
//
// - comments - "<!--comment-->" - are decoded as map["#comment"]map["#text"]"cmnt_text" with a "#seq" k:v pair.
//
// - directives - "<!text>" - are decoded as map["#directive"]map[#text"]"directive_text" with a "#seq" k:v pair.
//
// - process instructions - "<?instr?>" - are decoded as map["#procinst"]interface{} where the #procinst value
// is of map[string]interface{} type with the following keys: #target, #inst, and #seq.
//
// - comments, directives, and procinsts that are NOT part of a document with a root key will be returned as
// map[string]interface{} and the error value 'NoRoot'.
//
// - note: "<![CDATA[" syntax is lost in xml.Decode parser - and is not handled here, either.
// and: "\r\n" is converted to "\n"
//
// NOTES:
// 1. The 'xmlVal' will be parsed looking for an xml.StartElement, xml.Comment, etc., so BOM and other
// extraneous xml.CharData will be ignored unless io.EOF is reached first.
// 2. CoerceKeysToLower() is NOT recognized, since the intent here is to eventually call m.XmlSeq() to
// re-encode the message in its original structure.
// 3. If CoerceKeysToSnakeCase() has been called, then all key values will be converted to snake case.
//
// NAME SPACES:
// 1. Keys in the MapSeq value that are parsed from a <name space prefix>:<local name> tag preserve the
// "<prefix>:" notation rather than stripping it as with NewMapXml().
// 2. Attribute keys for name space prefix declarations preserve "xmlns:<prefix>" notation.
//
// ERRORS:
// 1. If a NoRoot error, "no root key," is returned, check the initial map key for a "#comment",
// "#directive" or #procinst" key.
// 2. Unmarshaling an XML doc that is formatted using the whitespace character, " ", will error, since
// Decoder.RawToken treats such occurances as significant. See NewMapFormattedXmlSeq().
func NewMapXmlSeq(xmlVal []byte, cast ...bool) (MapSeq, error) {
var r bool
if len(cast) == 1 {
r = cast[0]
}
return xmlSeqToMap(xmlVal, r)
}
// NewMapFormattedXmlSeq performs the same as NewMapXmlSeq but is useful for processing XML objects that
// are formatted using the whitespace character, " ". (The stdlib xml.Decoder, by default, treats all
// whitespace as significant; Decoder.Token() and Decoder.RawToken() will return strings of one or more
// whitespace characters and without alphanumeric or punctuation characters as xml.CharData values.)
//
// If you're processing such XML, then this will convert all occurrences of whitespace-only strings
// into an empty string, "", prior to parsing the XML - irrespective of whether the occurrence is
// formatting or is a actual element value.
func NewMapFormattedXmlSeq(xmlVal []byte, cast ...bool) (MapSeq, error) {
var c bool
if len(cast) == 1 {
c = cast[0]
}
// Per PR #104 - clean out formatting characters so they don't show up in Decoder.RawToken() stream.
// NOTE: Also replaces element values that are solely comprised of formatting/whitespace characters
// with empty string, "".
r := regexp.MustCompile(`>[\n\t\r ]*<`)
xmlVal = r.ReplaceAll(xmlVal, []byte("><"))
return xmlSeqToMap(xmlVal, c)
}
// NewMpaXmlSeqReader returns next XML doc from an io.Reader as a MapSeq value.
//
// NOTES:
// 1. The 'xmlReader' will be parsed looking for an xml.StartElement, xml.Comment, etc., so BOM and other
// extraneous xml.CharData will be ignored unless io.EOF is reached first.
// 2. CoerceKeysToLower() is NOT recognized, since the intent here is to eventually call m.XmlSeq() to
// re-encode the message in its original structure.
// 3. If CoerceKeysToSnakeCase() has been called, then all key values will be converted to snake case.
//
// ERRORS:
// 1. If a NoRoot error, "no root key," is returned, check the initial map key for a "#comment",
// "#directive" or #procinst" key.
func NewMapXmlSeqReader(xmlReader io.Reader, cast ...bool) (MapSeq, error) {
var r bool
if len(cast) == 1 {
r = cast[0]
}
// We need to put an *os.File reader in a ByteReader or the xml.NewDecoder
// will wrap it in a bufio.Reader and seek on the file beyond where the
// xml.Decoder parses!
if _, ok := xmlReader.(io.ByteReader); !ok {
xmlReader = myByteReader(xmlReader) // see code at EOF
}
// build the map
return xmlSeqReaderToMap(xmlReader, r)
}
// NewMapXmlSeqReaderRaw returns the next XML doc from an io.Reader as a MapSeq value.
// Returns MapSeq value, slice with the raw XML, and any error.
//
// NOTES:
// 1. Due to the implementation of xml.Decoder, the raw XML off the reader is buffered to []byte
// using a ByteReader. If the io.Reader is an os.File, there may be significant performance impact.
// See the examples - getmetrics1.go through getmetrics4.go - for comparative use cases on a large
// data set. If the io.Reader is wrapping a []byte value in-memory, however, such as http.Request.Body
// you CAN use it to efficiently unmarshal a XML doc and retrieve the raw XML in a single call.
// 2. The 'raw' return value may be larger than the XML text value.
// 3. The 'xmlReader' will be parsed looking for an xml.StartElement, xml.Comment, etc., so BOM and other
// extraneous xml.CharData will be ignored unless io.EOF is reached first.
// 4. CoerceKeysToLower() is NOT recognized, since the intent here is to eventually call m.XmlSeq() to
// re-encode the message in its original structure.
// 5. If CoerceKeysToSnakeCase() has been called, then all key values will be converted to snake case.
//
// ERRORS:
// 1. If a NoRoot error, "no root key," is returned, check if the initial map key is "#comment",
// "#directive" or #procinst" key.
func NewMapXmlSeqReaderRaw(xmlReader io.Reader, cast ...bool) (MapSeq, []byte, error) {
var r bool
if len(cast) == 1 {
r = cast[0]
}
// create TeeReader so we can retrieve raw XML
buf := make([]byte, 0)
wb := bytes.NewBuffer(buf)
trdr := myTeeReader(xmlReader, wb)
m, err := xmlSeqReaderToMap(trdr, r)
// retrieve the raw XML that was decoded
b := wb.Bytes()
// err may be NoRoot
return m, b, err
}
// xmlSeqReaderToMap() - parse a XML io.Reader to a map[string]interface{} value
func xmlSeqReaderToMap(rdr io.Reader, r bool) (map[string]interface{}, error) {
// parse the Reader
p := xml.NewDecoder(rdr)
if CustomDecoder != nil {
useCustomDecoder(p)
} else {
p.CharsetReader = XmlCharsetReader
}
return xmlSeqToMapParser("", nil, p, r)
}
// xmlSeqToMap - convert a XML doc into map[string]interface{} value
func xmlSeqToMap(doc []byte, r bool) (map[string]interface{}, error) {
b := bytes.NewReader(doc)
p := xml.NewDecoder(b)
if CustomDecoder != nil {
useCustomDecoder(p)
} else {
p.CharsetReader = XmlCharsetReader
}
return xmlSeqToMapParser("", nil, p, r)
}
// ===================================== where the work happens =============================
// xmlSeqToMapParser - load a 'clean' XML doc into a map[string]interface{} directly.
// Add #seq tag value for each element decoded - to be used for Encoding later.
func xmlSeqToMapParser(skey string, a []xml.Attr, p *xml.Decoder, r bool) (map[string]interface{}, error) {
if snakeCaseKeys {
skey = strings.Replace(skey, "-", "_", -1)
}
// NOTE: all attributes and sub-elements parsed into 'na', 'na' is returned as value for 'skey' in 'n'.
var n, na map[string]interface{}
var seq int // for including seq num when decoding
// Allocate maps and load attributes, if any.
// NOTE: on entry from NewMapXml(), etc., skey=="", and we fall through
// to get StartElement then recurse with skey==xml.StartElement.Name.Local
// where we begin allocating map[string]interface{} values 'n' and 'na'.
if skey != "" {
// 'n' only needs one slot - save call to runtime•hashGrow()
// 'na' we don't know
n = make(map[string]interface{}, 1)
na = make(map[string]interface{})
if len(a) > 0 {
// xml.Attr is decoded into: map["#attr"]map[<attr_label>]interface{}
// where interface{} is map[string]interface{}{"#text":<attr_val>, "#seq":<attr_seq>}
aa := make(map[string]interface{}, len(a))
for i, v := range a {
if snakeCaseKeys {
v.Name.Local = strings.Replace(v.Name.Local, "-", "_", -1)
}
if xmlEscapeCharsDecoder { // per issue#84
v.Value = escapeChars(v.Value)
}
if len(v.Name.Space) > 0 {
aa[v.Name.Space+`:`+v.Name.Local] = map[string]interface{}{textK: cast(v.Value, r, ""), seqK: i}
} else {
aa[v.Name.Local] = map[string]interface{}{textK: cast(v.Value, r, ""), seqK: i}
}
}
na[attrK] = aa
}
}
// Return XMPP <stream:stream> message.
if handleXMPPStreamTag && skey == "stream:stream" {
n[skey] = na
return n, nil
}
for {
t, err := p.RawToken()
if err != nil {
if err != io.EOF {
return nil, errors.New("xml.Decoder.Token() - " + err.Error())
}
return nil, err
}
switch t.(type) {
case xml.StartElement:
tt := t.(xml.StartElement)
// First call to xmlSeqToMapParser() doesn't pass xml.StartElement - the map key.
// So when the loop is first entered, the first token is the root tag along
// with any attributes, which we process here.
//
// Subsequent calls to xmlSeqToMapParser() will pass in tag+attributes for
// processing before getting the next token which is the element value,
// which is done above.
if skey == "" {
if len(tt.Name.Space) > 0 {
return xmlSeqToMapParser(tt.Name.Space+`:`+tt.Name.Local, tt.Attr, p, r)
} else {
return xmlSeqToMapParser(tt.Name.Local, tt.Attr, p, r)
}
}
// If not initializing the map, parse the element.
// len(nn) == 1, necessarily - it is just an 'n'.
var nn map[string]interface{}
if len(tt.Name.Space) > 0 {
nn, err = xmlSeqToMapParser(tt.Name.Space+`:`+tt.Name.Local, tt.Attr, p, r)
} else {
nn, err = xmlSeqToMapParser(tt.Name.Local, tt.Attr, p, r)
}
if err != nil {
return nil, err
}
// The nn map[string]interface{} value is a na[nn_key] value.
// We need to see if nn_key already exists - means we're parsing a list.
// This may require converting na[nn_key] value into []interface{} type.
// First, extract the key:val for the map - it's a singleton.
var key string
var val interface{}
for key, val = range nn {
break
}
// add "#seq" k:v pair -
// Sequence number included even in list elements - this should allow us
// to properly resequence even something goofy like:
// <list>item 1</list>
// <subelement>item 2</subelement>
// <list>item 3</list>
// where all the "list" subelements are decoded into an array.
switch val.(type) {
case map[string]interface{}:
val.(map[string]interface{})[seqK] = seq
seq++
case interface{}: // a non-nil simple element: string, float64, bool
v := map[string]interface{}{textK: val, seqK: seq}
seq++
val = v
}
// 'na' holding sub-elements of n.
// See if 'key' already exists.
// If 'key' exists, then this is a list, if not just add key:val to na.
if v, ok := na[key]; ok {
var a []interface{}
switch v.(type) {
case []interface{}:
a = v.([]interface{})
default: // anything else - note: v.(type) != nil
a = []interface{}{v}
}
a = append(a, val)
na[key] = a
} else {
na[key] = val // save it as a singleton
}
case xml.EndElement:
if skey != "" {
tt := t.(xml.EndElement)
if snakeCaseKeys {
tt.Name.Local = strings.Replace(tt.Name.Local, "-", "_", -1)
}
var name string
if len(tt.Name.Space) > 0 {
name = tt.Name.Space + `:` + tt.Name.Local
} else {
name = tt.Name.Local
}
if skey != name {
return nil, fmt.Errorf("element %s not properly terminated, got %s at #%d",
skey, name, p.InputOffset())
}
}
// len(n) > 0 if this is a simple element w/o xml.Attrs - see xml.CharData case.
if len(n) == 0 {
// If len(na)==0 we have an empty element == "";
// it has no xml.Attr nor xml.CharData.
// Empty element content will be map["etag"]map["#text"]""
// after #seq injection - map["etag"]map["#seq"]seq - after return.
if len(na) > 0 {
n[skey] = na
} else {
n[skey] = "" // empty element
}
}
return n, nil
case xml.CharData:
// clean up possible noise
tt := strings.Trim(string(t.(xml.CharData)), trimRunes)
if xmlEscapeCharsDecoder { // issue#84
tt = escapeChars(tt)
}
if skey == "" {
// per Adrian (http://www.adrianlungu.com/) catch stray text
// in decoder stream -
// https://github.com/clbanning/mxj/pull/14#issuecomment-182816374
// NOTE: CharSetReader must be set to non-UTF-8 CharSet or you'll get
// a p.Token() decoding error when the BOM is UTF-16 or UTF-32.
continue
}
if len(tt) > 0 {
// every simple element is a #text and has #seq associated with it
na[textK] = cast(tt, r, "")
na[seqK] = seq
seq++
}
case xml.Comment:
if n == nil { // no root 'key'
n = map[string]interface{}{commentK: string(t.(xml.Comment))}
return n, NoRoot
}
cm := make(map[string]interface{}, 2)
cm[textK] = string(t.(xml.Comment))
cm[seqK] = seq
seq++
na[commentK] = cm
case xml.Directive:
if n == nil { // no root 'key'
n = map[string]interface{}{directiveK: string(t.(xml.Directive))}
return n, NoRoot
}
dm := make(map[string]interface{}, 2)
dm[textK] = string(t.(xml.Directive))
dm[seqK] = seq
seq++
na[directiveK] = dm
case xml.ProcInst:
if n == nil {
na = map[string]interface{}{targetK: t.(xml.ProcInst).Target, instK: string(t.(xml.ProcInst).Inst)}
n = map[string]interface{}{procinstK: na}
return n, NoRoot
}
pm := make(map[string]interface{}, 3)
pm[targetK] = t.(xml.ProcInst).Target
pm[instK] = string(t.(xml.ProcInst).Inst)
pm[seqK] = seq
seq++
na[procinstK] = pm
default:
// noop - shouldn't ever get here, now, since we handle all token types
}
}
}
// ------------------ END: NewMapXml & NewMapXmlReader -------------------------
// --------------------- mv.XmlSeq & mv.XmlSeqWriter -------------------------
// Xml encodes a MapSeq as XML with elements sorted on #seq. The companion of NewMapXmlSeq().
// The following rules apply.
// - The "#seq" key value is used to seqence the subelements or attributes only.
// - The "#attr" map key identifies the map of attribute map[string]interface{} values with "#text" key.
// - The "#comment" map key identifies a comment in the value "#text" map entry - <!--comment-->.
// - The "#directive" map key identifies a directive in the value "#text" map entry - <!directive>.
// - The "#procinst" map key identifies a process instruction in the value "#target" and "#inst"
// map entries - <?target inst?>.
// - Value type encoding:
// > string, bool, float64, int, int32, int64, float32: per "%v" formating
// > []bool, []uint8: by casting to string
// > structures, etc.: handed to xml.Marshal() - if there is an error, the element
// value is "UNKNOWN"
// - Elements with only attribute values or are null are terminated using "/>" unless XmlGoEmptyElemSystax() called.
// - If len(mv) == 1 and no rootTag is provided, then the map key is used as the root tag, possible.
// Thus, `{ "key":"value" }` encodes as "<key>value</key>".
func (mv MapSeq) Xml(rootTag ...string) ([]byte, error) {
m := map[string]interface{}(mv)
var err error
s := new(string)
p := new(pretty) // just a stub
var sb strings.Builder
if len(m) == 1 && len(rootTag) == 0 {
for key, value := range m {
// if it's an array, see if all values are map[string]interface{}
// we force a new root tag if we'll end up with no key:value in the list
// so: key:[string_val, bool:true] --> <doc><key>string_val</key><bool>true</bool></doc>
switch value.(type) {
case []interface{}:
for _, v := range value.([]interface{}) {
switch v.(type) {
case map[string]interface{}: // noop
default: // anything else
err = mapToXmlSeqIndent(false, &sb, DefaultRootTag, m, p)
goto done
}
}
}
err = mapToXmlSeqIndent(false, &sb, key, value, p)
}
} else if len(rootTag) == 1 {
err = mapToXmlSeqIndent(false, &sb, rootTag[0], m, p)
} else {
err = mapToXmlSeqIndent(false, &sb, DefaultRootTag, m, p)
}
done:
if xmlCheckIsValid {
d := xml.NewDecoder(bytes.NewReader([]byte(*s)))
for {
_, err = d.Token()
if err == io.EOF {
err = nil
break
} else if err != nil {
return nil, err
}
}
}
return []byte(sb.String()), err
}
// The following implementation is provided only for symmetry with NewMapXmlReader[Raw]
// The names will also provide a key for the number of return arguments.
// XmlWriter Writes the MapSeq value as XML on the Writer.
// See MapSeq.Xml() for encoding rules.
func (mv MapSeq) XmlWriter(xmlWriter io.Writer, rootTag ...string) error {
x, err := mv.Xml(rootTag...)
if err != nil {
return err
}
_, err = xmlWriter.Write(x)
return err
}
// XmlWriteRaw writes the MapSeq value as XML on the Writer. []byte is the raw XML that was written.
// See Map.XmlSeq() for encoding rules.
/*
func (mv MapSeq) XmlWriterRaw(xmlWriter io.Writer, rootTag ...string) ([]byte, error) {
x, err := mv.Xml(rootTag...)
if err != nil {
return x, err
}
_, err = xmlWriter.Write(x)
return x, err
}
*/
// XmlIndentWriter writes the MapSeq value as pretty XML on the Writer.
// See MapSeq.Xml() for encoding rules.
func (mv MapSeq) XmlIndentWriter(xmlWriter io.Writer, prefix, indent string, rootTag ...string) error {
x, err := mv.XmlIndent(prefix, indent, rootTag...)
if err != nil {
return err
}
_, err = xmlWriter.Write(x)
return err
}
// XmlIndentWriterRaw writes the Map as pretty XML on the Writer. []byte is the raw XML that was written.
// See Map.XmlSeq() for encoding rules.
/*
func (mv MapSeq) XmlIndentWriterRaw(xmlWriter io.Writer, prefix, indent string, rootTag ...string) ([]byte, error) {
x, err := mv.XmlSeqIndent(prefix, indent, rootTag...)
if err != nil {
return x, err
}
_, err = xmlWriter.Write(x)
return x, err
}
*/
// -------------------- END: mv.Xml & mv.XmlWriter -------------------------------
// ---------------------- XmlSeqIndent ----------------------------
// XmlIndent encodes a map[string]interface{} as a pretty XML string.
// See MapSeq.XmlSeq() for encoding rules.
func (mv MapSeq) XmlIndent(prefix, indent string, rootTag ...string) ([]byte, error) {
m := map[string]interface{}(mv)
var err error
p := new(pretty)
p.indent = indent
p.padding = prefix
var sb strings.Builder
if len(m) == 1 && len(rootTag) == 0 {
// this can extract the key for the single map element
// use it if it isn't a key for a list
for key, value := range m {
if _, ok := value.([]interface{}); ok {
err = mapToXmlSeqIndent(true, &sb, DefaultRootTag, m, p)
} else {
err = mapToXmlSeqIndent(true, &sb, key, value, p)
}
}
} else if len(rootTag) == 1 {
err = mapToXmlSeqIndent(true, &sb, rootTag[0], m, p)
} else {
err = mapToXmlSeqIndent(true, &sb, DefaultRootTag, m, p)
}
if xmlCheckIsValid {
if _, err = NewMapXml([]byte(sb.String())); err != nil {
return nil, err
}
d := xml.NewDecoder(bytes.NewReader([]byte(sb.String())))
for {
_, err = d.Token()
if err == io.EOF {
err = nil
break
} else if err != nil {
return nil, err
}
}
}
return []byte(sb.String()), err
}
// where the work actually happens
// returns an error if an attribute is not atomic
func mapToXmlSeqIndent(doIndent bool, sb *strings.Builder, key string, value interface{}, pp *pretty) error {
var endTag bool
var isSimple bool
var noEndTag bool
var elen int
var ss string
p := &pretty{pp.indent, pp.cnt, pp.padding, pp.mapDepth, pp.start}
switch value.(type) {
case map[string]interface{}, []byte, string, float64, bool, int, int32, int64, float32:
if doIndent {
sb.WriteString(p.padding)
}
if key != commentK && key != directiveK && key != procinstK {
sb.WriteString("<")
sb.WriteString(key)
}
}
switch value.(type) {
case map[string]interface{}:
val := value.(map[string]interface{})
if key == commentK {
sb.WriteString("<!--")
sb.WriteString(val[textK].(string))
sb.WriteString("-->")
noEndTag = true
break
}
if key == directiveK {
sb.WriteString("<!")
sb.WriteString(val[textK].(string))
sb.WriteString(">")
noEndTag = true
break
}
if key == procinstK {
sb.WriteString("<?")
sb.WriteString(val[targetK].(string))
sb.WriteString(" ")
sb.WriteString(val[instK].(string))
sb.WriteString("?>")
noEndTag = true
break
}
haveAttrs := false
// process attributes first
if v, ok := val[attrK].(map[string]interface{}); ok {
// First, unroll the map[string]interface{} into a []keyval array.
// Then sequence it.
kv := make([]keyval, len(v))
n := 0
for ak, av := range v {
kv[n] = keyval{ak, av}
n++
}
sort.Sort(elemListSeq(kv))
// Now encode the attributes in original decoding sequence, using keyval array.
for _, a := range kv {
vv := a.v.(map[string]interface{})
switch vv[textK].(type) {
case string:
if xmlEscapeChars {
ss = escapeChars(vv[textK].(string))
} else {
ss = vv[textK].(string)
}
sb.WriteString(" ")
sb.WriteString(a.k)
sb.WriteString(`="`)
sb.WriteString(ss)
sb.WriteString(`"`)
case float64, bool, int, int32, int64, float32:
sb.WriteString(" ")
sb.WriteString(a.k)
sb.WriteString(`="`)
sb.WriteString(fmt.Sprintf("%v", vv[textK]))
sb.WriteString(`"`)
case []byte:
if xmlEscapeChars {
ss = escapeChars(string(vv[textK].([]byte)))
} else {
ss = string(vv[textK].([]byte))
}
sb.WriteString(" ")
sb.WriteString(a.k)
sb.WriteString(`="`)
sb.WriteString(ss)
sb.WriteString(`"`)
default:
return fmt.Errorf("invalid attribute value for: %s", a.k)
}
}
haveAttrs = true
}
// simple element?
// every map value has, at least, "#seq" and, perhaps, "#text" and/or "#attr"
_, seqOK := val[seqK] // have key
if v, ok := val[textK]; ok && ((len(val) == 3 && haveAttrs) || (len(val) == 2 && !haveAttrs)) && seqOK {
if stmp, ok := v.(string); ok && stmp != "" {
if xmlEscapeChars {
stmp = escapeChars(stmp)
}
sb.WriteString(">")
sb.WriteString(stmp)
endTag = true
elen = 1
}
isSimple = true
break
} else if !ok && ((len(val) == 2 && haveAttrs) || (len(val) == 1 && !haveAttrs)) && seqOK {
// here no #text but have #seq or #seq+#attr
endTag = false
break
}
// we now need to sequence everything except attributes
// 'kv' will hold everything that needs to be written
kv := make([]keyval, 0)
for k, v := range val {
if k == attrK { // already processed
continue
}
if k == seqK { // ignore - just for sorting
continue
}
switch v.(type) {
case []interface{}:
// unwind the array as separate entries
for _, vv := range v.([]interface{}) {
kv = append(kv, keyval{k, vv})
}
default:
kv = append(kv, keyval{k, v})
}
}
// close tag with possible attributes
sb.WriteString(">")
if doIndent {
sb.WriteString("\n")
}
// something more complex
p.mapDepth++
sort.Sort(elemListSeq(kv))
i := 0
for _, v := range kv {
switch v.v.(type) {
case []interface{}:
default:
if i == 0 && doIndent {
p.Indent()
}
}
i++
if err := mapToXmlSeqIndent(doIndent, sb, v.k, v.v, p); err != nil {
return err
}
switch v.v.(type) {
case []interface{}: // handled in []interface{} case
default:
if doIndent {
p.Outdent()
}
}
i--
}
p.mapDepth--
endTag = true
elen = 1 // we do have some content other than attrs
case []interface{}:
for _, v := range value.([]interface{}) {
if doIndent {
p.Indent()
}
if err := mapToXmlSeqIndent(doIndent, sb, key, v, p); err != nil {
return err
}
if doIndent {
p.Outdent()
}
}
return nil
case nil:
// terminate the tag
if doIndent {
sb.WriteString(p.padding)
}
sb.WriteString("<")
sb.WriteString(key)
endTag, isSimple = true, true
break
default: // handle anything - even goofy stuff
elen = 0
switch value.(type) {
case string:
if xmlEscapeChars {
ss = escapeChars(value.(string))
} else {
ss = value.(string)
}
elen = len(ss)
if elen > 0 {
sb.WriteString(">")
sb.WriteString(ss)
}
case float64, bool, int, int32, int64, float32:
v := fmt.Sprintf("%v", value)
elen = len(v)
if elen > 0 {
sb.WriteString(">")
sb.WriteString(v)
}
case []byte: // NOTE: byte is just an alias for uint8
// similar to how xml.Marshal handles []byte structure members
if xmlEscapeChars {
ss = escapeChars(string(value.([]byte)))
} else {
ss = string(value.([]byte))
}
elen = len(ss)
if elen > 0 {
sb.WriteString(">")
sb.WriteString(ss)
}
default:
var v []byte
var err error
if doIndent {
v, err = xml.MarshalIndent(value, p.padding, p.indent)
} else {
v, err = xml.Marshal(value)
}
if err != nil {
sb.WriteString(">UNKNOWN")
} else {
elen = len(v)
if elen > 0 {
sb.WriteString(string(v))
}
}
}
isSimple = true
endTag = true
}
if endTag && !noEndTag {
if doIndent {
if !isSimple {
sb.WriteString(p.padding)
}
}
switch value.(type) {
case map[string]interface{}, []byte, string, float64, bool, int, int32, int64, float32:
if elen > 0 || useGoXmlEmptyElemSyntax {
if elen == 0 {
sb.WriteString(">")
}
sb.WriteString("</")
sb.WriteString(key)
sb.WriteString(">")
} else {
sb.WriteString("/>")
}
}
} else if !noEndTag {
if useGoXmlEmptyElemSyntax {
sb.WriteString("</")
sb.WriteString(key)
sb.WriteString(">")
// *s += "></" + key + ">"
} else {
sb.WriteString("/>")
}
}
if doIndent {
if p.cnt > p.start {
sb.WriteString("\n")
}
p.Outdent()
}
return nil
}
// the element sort implementation
type keyval struct {
k string
v interface{}
}
type elemListSeq []keyval
func (e elemListSeq) Len() int {
return len(e)
}
func (e elemListSeq) Swap(i, j int) {
e[i], e[j] = e[j], e[i]
}
func (e elemListSeq) Less(i, j int) bool {
var iseq, jseq int
var fiseq, fjseq float64
var ok bool
if iseq, ok = e[i].v.(map[string]interface{})[seqK].(int); !ok {
if fiseq, ok = e[i].v.(map[string]interface{})[seqK].(float64); ok {
iseq = int(fiseq)
} else {
iseq = 9999999
}
}
if jseq, ok = e[j].v.(map[string]interface{})[seqK].(int); !ok {
if fjseq, ok = e[j].v.(map[string]interface{})[seqK].(float64); ok {
jseq = int(fjseq)
} else {
jseq = 9999999
}
}
return iseq <= jseq
}
// =============== https://groups.google.com/forum/#!topic/golang-nuts/lHPOHD-8qio
// BeautifyXml (re)formats an XML doc similar to Map.XmlIndent().
// It preserves comments, directives and process instructions,
func BeautifyXml(b []byte, prefix, indent string) ([]byte, error) {
x, err := NewMapXmlSeq(b)
if err != nil {
return nil, err
}
return x.XmlIndent(prefix, indent)
}
// Copyright 2012-2016, 2019 Charles Banning. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file
package mxj
// ---------------- expose Map methods to MapSeq type ---------------------------
// Pretty print a Map.
func (msv MapSeq) StringIndent(offset ...int) string {
return writeMap(map[string]interface{}(msv), true, true, offset...)
}
// Pretty print a Map without the value type information - just key:value entries.
func (msv MapSeq) StringIndentNoTypeInfo(offset ...int) string {
return writeMap(map[string]interface{}(msv), false, true, offset...)
}