Can add string to scalars in any order #1234

* Add xml-keep-namespace and xml-raw-token features

* Add tests

* Change flags usage strings

* Append docs

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.18.2 as builder
+FROM golang:1.18.3 as builder
 
 WORKDIR /go/src/mikefarah/yq
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.18.2
+FROM golang:1.18.3
 
 COPY scripts/devtools.sh /opt/devtools.sh
 

cmd/constant.go

@@ -11,6 +11,8 @@ var inputFormat = "yaml"
 var xmlAttributePrefix = "+"
 var xmlContentName = "+content"
 var xmlStrictMode = false
+var xmlKeepNamespace = false
+var xmlUseRawToken = false
 
 var exitStatus = false
 var forceColor = false

cmd/root.go

@@ -72,6 +72,8 @@ yq -P sample.json
 	rootCmd.PersistentFlags().StringVar(&xmlAttributePrefix, "xml-attribute-prefix", "+", "prefix for xml attributes")
 	rootCmd.PersistentFlags().StringVar(&xmlContentName, "xml-content-name", "+content", "name for xml content (if no attribute name is present).")
 	rootCmd.PersistentFlags().BoolVar(&xmlStrictMode, "xml-strict-mode", false, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
+	rootCmd.PersistentFlags().BoolVar(&xmlKeepNamespace, "xml-keep-namespace", false, "enables keeping namespace after parsing attributes")
+	rootCmd.PersistentFlags().BoolVar(&xmlUseRawToken, "xml-raw-token", false, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
 
 	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")

cmd/utils.go

@@ -63,7 +63,7 @@ func configureDecoder() (yqlib.Decoder, error) {
 	}
 	switch yqlibInputFormat {
 	case yqlib.XMLInputFormat:
-		return yqlib.NewXMLDecoder(xmlAttributePrefix, xmlContentName, xmlStrictMode), nil
+		return yqlib.NewXMLDecoder(xmlAttributePrefix, xmlContentName, xmlStrictMode, xmlKeepNamespace, xmlUseRawToken), nil
 	case yqlib.PropertiesInputFormat:
 		return yqlib.NewPropertiesDecoder(), nil
 	}

examples/data1.yaml

@@ -1,2 +1,12 @@
-a: mike
+# --------------------------------------------------
-b: [t, f]
+# It's a test with comment
+# --------------------------------------------------
+test:
+  - name: a
+  - name: b
+  - name: c
+groups:
+  - name: a
+  - name: b
+  - name: c
+  - name: c

examples/data2.yaml

@@ -1,2 +1,5 @@
-c:
+# --------------------------------------------------
-  d: hamster
+# It's a test with comment
+# --------------------------------------------------
+groups:
+  - name: d

go.sum

@@ -72,7 +72,5 @@ gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/yqlib/candidate_node.go

@@ -86,18 +86,24 @@ func (n *CandidateNode) CreateChildInArray(index int, node *yaml.Node) *Candidat
 
 func (n *CandidateNode) CreateReplacement(node *yaml.Node) *CandidateNode {
 	return &CandidateNode{
-		Node:           node,
+		Node:      node,
-		Path:           n.createChildPath(nil),
+		Path:      n.createChildPath(nil),
-		LeadingContent: n.LeadingContent,
+		Parent:    n.Parent,
-		Parent:         n.Parent,
+		Key:       n.Key,
-		Key:            n.Key,
+		IsMapKey:  n.IsMapKey,
-		IsMapKey:       n.IsMapKey,
+		Document:  n.Document,
-		Document:       n.Document,
+		Filename:  n.Filename,
-		Filename:       n.Filename,
+		FileIndex: n.FileIndex,
-		FileIndex:      n.FileIndex,
 	}
 }
 
+func (n *CandidateNode) CreateReplacementWithDocWrappers(node *yaml.Node) *CandidateNode {
+	replacement := n.CreateReplacement(node)
+	replacement.LeadingContent = n.LeadingContent
+	replacement.TrailingContent = n.TrailingContent
+	return replacement
+}
+
 func (n *CandidateNode) createChildPath(path interface{}) []interface{} {
 	if path == nil {
 		newPath := make([]interface{}, len(n.Path))

pkg/yqlib/decoder_xml.go

@@ -17,14 +17,23 @@ type xmlDecoder struct {
 	attributePrefix string
 	contentName     string
 	strictMode      bool
+	keepNamespace   bool
+	useRawToken     bool
 	finished        bool
 }
 
-func NewXMLDecoder(attributePrefix string, contentName string, strictMode bool) Decoder {
+func NewXMLDecoder(attributePrefix string, contentName string, strictMode bool, keepNamespace bool, useRawToken bool) Decoder {
 	if contentName == "" {
 		contentName = "content"
 	}
-	return &xmlDecoder{attributePrefix: attributePrefix, contentName: contentName, finished: false, strictMode: strictMode}
+	return &xmlDecoder{
+		attributePrefix: attributePrefix,
+		contentName:     contentName,
+		finished:        false,
+		strictMode:      strictMode,
+		keepNamespace:   keepNamespace,
+		useRawToken:     useRawToken,
+	}
 }
 
 func (dec *xmlDecoder) Init(reader io.Reader) {
@@ -206,8 +215,15 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 		n:      root,
 	}
 
+	getToken := func() (xml.Token, error) {
+		if dec.useRawToken {
+			return xmlDec.RawToken()
+		}
+		return xmlDec.Token()
+	}
+
 	for {
-		t, e := xmlDec.Token()
+		t, e := getToken()
 		if e != nil && !errors.Is(e, io.EOF) {
 			return e
 		}
@@ -228,6 +244,11 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 
 			// Extract attributes as children
 			for _, a := range se.Attr {
+				if dec.keepNamespace {
+					if a.Name.Space != "" {
+						a.Name.Local = a.Name.Space + ":" + a.Name.Local
+					}
+				}
 				elem.n.AddChild(dec.attributePrefix+a.Name.Local, &xmlNode{Data: a.Value})
 			}
 		case xml.CharData:

pkg/yqlib/doc/usage/xml.md

@@ -192,6 +192,52 @@ cat:
 # after cat
 ```
 
+## Parse xml: keep attribute namespace
+Given a sample.xml file of:
+```xml
+
+<?xml version="1.0"?>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
+</map>
+
+```
+then
+```bash
+yq -p=xml -o=xml --xml-keep-namespace '.' sample.xml
+```
+will output
+```xml
+<map xmlns="some-namespace" xmlns:xsi="some-instance" some-instance:schemaLocation="some-url"></map>
+```
+
+instead of
+```xml
+<map xmlns="some-namespace" xsi="some-instance" schemaLocation="some-url"></map>
+```
+
+## Parse xml: keep raw attribute namespace
+Given a sample.xml file of:
+```xml
+
+<?xml version="1.0"?>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
+</map>
+
+```
+then
+```bash
+yq -p=xml -o=xml --xml-keep-namespace --xml-raw-token '.' sample.xml
+```
+will output
+```xml
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url"></map>
+```
+
+instead of
+```xml
+<map xmlns="some-namespace" xsi="some-instance" schemaLocation="some-url"></map>
+```
+
 ## Encode xml: simple
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/encoder_xml.go

@@ -9,7 +9,7 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-var XMLPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content", StrictMode: false}
+var XMLPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content", StrictMode: false, UseRawToken: false}
 
 type xmlEncoder struct {
 	attributePrefix string

pkg/yqlib/expression_tokeniser.go

@@ -420,9 +420,9 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte(`load`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewYamlDecoder()}))
 
-	lexer.Add([]byte(`xmlload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode)}))
+	lexer.Add([]byte(`xmlload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode, XMLPreferences.KeepNamespace, XMLPreferences.UseRawToken)}))
-	lexer.Add([]byte(`load_xml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode)}))
+	lexer.Add([]byte(`load_xml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode, XMLPreferences.KeepNamespace, XMLPreferences.UseRawToken)}))
-	lexer.Add([]byte(`loadxml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode)}))
+	lexer.Add([]byte(`loadxml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode, XMLPreferences.KeepNamespace, XMLPreferences.UseRawToken)}))
 
 	lexer.Add([]byte(`load_base64`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewBase64Decoder()}))
 

pkg/yqlib/lib.go

@@ -26,6 +26,8 @@ type xmlPreferences struct {
 	AttributePrefix string
 	ContentName     string
 	StrictMode      bool
+	KeepNamespace   bool
+	UseRawToken     bool
 }
 
 var log = logging.MustGetLogger("yq-lib")

pkg/yqlib/operator_add.go

@@ -103,6 +103,9 @@ func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yam
 	} else if lhsTag == "!!str" {
 		target.Node.Tag = lhs.Tag
 		target.Node.Value = lhs.Value + rhs.Value
+	} else if rhsTag == "!!str" {
+		target.Node.Tag = rhs.Tag
+		target.Node.Value = lhs.Value + rhs.Value
 	} else if lhsTag == "!!int" && rhsTag == "!!int" {
 		format, lhsNum, err := parseInt64(lhs.Value)
 		if err != nil {

pkg/yqlib/operator_add_test.go

@@ -199,6 +199,24 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: catmeow, b: meow}\n",
 		},
 	},
+	{
+		description: "String concatenation - str + int",
+		skipDoc:     true,
+		document:    `{a: !cool cat, b: meow}`,
+		expression:  `.a + 3`,
+		expected: []string{
+			"D0, P[a], (!cool)::cat3\n",
+		},
+	},
+	{
+		description: "String concatenation - int + str",
+		skipDoc:     true,
+		document:    `{a: !cool cat, b: meow}`,
+		expression:  `3 + .a`,
+		expected: []string{
+			"D0, P[], (!cool)::3cat\n",
+		},
+	},
 	{
 		description:    "Number addition - float",
 		subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.",

pkg/yqlib/operator_collect_test.go

@@ -29,14 +29,23 @@ var collectOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		skipDoc:  true,
+		skipDoc:    true,
-		document: "{a: apple}\n---\n{b: frog}",
+		document:   "{a: apple}\n---\n{b: frog}",
-
 		expression: `[.]`,
 		expected: []string{
 			"D0, P[], (!!seq)::- {a: apple}\n- {b: frog}\n",
 		},
 	},
+	{
+		description: "with comments",
+		skipDoc:     true,
+		document:    "# abc\n[{a: apple}]\n\n# xyz\n",
+
+		expression: `[.[]]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- {a: apple}\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   ``,

pkg/yqlib/operator_encoder_decoder.go

@@ -104,7 +104,12 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 	case YamlInputFormat:
 		decoder = NewYamlDecoder()
 	case XMLInputFormat:
-		decoder = NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode)
+		decoder = NewXMLDecoder(
+			XMLPreferences.AttributePrefix,
+			XMLPreferences.ContentName,
+			XMLPreferences.StrictMode,
+			XMLPreferences.KeepNamespace,
+			XMLPreferences.UseRawToken)
 	case Base64InputFormat:
 		decoder = NewBase64Decoder()
 	case PropertiesInputFormat:

pkg/yqlib/operator_entries.go

@@ -20,7 +20,7 @@ func entrySeqFor(key *yaml.Node, value *yaml.Node) *yaml.Node {
 
 func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {
 	var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
-	var entriesNode = candidateNode.CreateReplacement(sequence)
+	var entriesNode = candidateNode.CreateReplacementWithDocWrappers(sequence)
 
 	var contents = unwrapDoc(candidateNode.Node).Content
 	for index := 0; index < len(contents); index = index + 2 {
@@ -34,7 +34,7 @@ func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {
 
 func toEntriesfromSeq(candidateNode *CandidateNode) *CandidateNode {
 	var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
-	var entriesNode = candidateNode.CreateReplacement(sequence)
+	var entriesNode = candidateNode.CreateReplacementWithDocWrappers(sequence)
 
 	var contents = unwrapDoc(candidateNode.Node).Content
 	for index := 0; index < len(contents); index = index + 1 {
@@ -94,7 +94,7 @@ func parseEntry(entry *yaml.Node, position int) (*yaml.Node, *yaml.Node, error)
 
 func fromEntries(candidateNode *CandidateNode) (*CandidateNode, error) {
 	var node = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
-	var mapCandidateNode = candidateNode.CreateReplacement(node)
+	var mapCandidateNode = candidateNode.CreateReplacementWithDocWrappers(node)
 
 	var contents = unwrapDoc(candidateNode.Node).Content
 
@@ -141,9 +141,11 @@ func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *
 	var results = list.New()
 
 	for el := toEntries.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
 		//run expression against entries
 		// splat toEntries and pipe it into Rhs
-		splatted, err := splat(context.SingleChildContext(el.Value.(*CandidateNode)), traversePreferences{})
+		splatted, err := splat(context.SingleChildContext(candidate), traversePreferences{})
 		if err != nil {
 			return Context{}, err
 		}
@@ -160,6 +162,10 @@ func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *
 		if err != nil {
 			return Context{}, err
 		}
+		collected.LeadingContent = candidate.LeadingContent
+		collected.TrailingContent = candidate.TrailingContent
+
+		log.Debugf("**** collected %v", collected.LeadingContent)
 
 		fromEntries, err := fromEntriesOperator(d, context.SingleChildContext(collected), expressionNode)
 		if err != nil {

pkg/yqlib/operator_entries_test.go

@@ -62,6 +62,15 @@ var entriesOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::KEY_c: 1\nKEY_d: 2\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `[{a: 1, b: 2}, {c: 1, d: 2}]`,
+		expression: `.[] | with_entries(.key |= "KEY_" + .)`,
+		expected: []string{
+			"D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n",
+			"D0, P[], (!!map)::KEY_c: 1\nKEY_d: 2\n",
+		},
+	},
 	{
 		description: "Use with_entries to filter the map",
 		document:    `{a: { b: bird }, c: { d: dog }}`,
@@ -70,6 +79,15 @@ var entriesOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::a: {b: bird}\n",
 		},
 	},
+	{
+		description: "Use with_entries to filter the map; head comment",
+		skipDoc:     true,
+		document:    "# abc\n{a: { b: bird }, c: { d: dog }}\n# xyz",
+		expression:  `with_entries(select(.value | has("b")))`,
+		expected: []string{
+			"D0, P[], (!!map)::# abc\na: {b: bird}\n# xyz\n",
+		},
+	},
 }
 
 func TestEntriesOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_env_test.go

@@ -150,6 +150,16 @@ var envOperatorScenarios = []expressionScenario{
 		expression:    `"the ${notThere} ${alsoNotThere}" | envsubst(nu,ff)`,
 		expectedError: "variable ${notThere} not set",
 	},
+	{
+		description:          "with header/footer",
+		skipDoc:              true,
+		environmentVariables: map[string]string{"myenv": "cat meow"},
+		document:             "# abc\n{v: \"${myenv}\"}\n# xyz\n",
+		expression:           `(.. | select(tag == "!!str")) |= envsubst`,
+		expected: []string{
+			"D0, P[], (!!map)::# abc\n{v: \"cat meow\"}\n# xyz\n",
+		},
+	},
 }
 
 func TestEnvOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_length_test.go

@@ -32,7 +32,7 @@ var lengthOperatorScenarios = []expressionScenario{
 	},
 	{
 		skipDoc:    true,
-		document:   `{a: key no exist}`,
+		document:   "# abc\n{a: key no exist}",
 		expression: `.b | length`,
 		expected: []string{
 			"D0, P[b], (!!int)::0\n",

pkg/yqlib/operator_pick.go

@@ -79,7 +79,7 @@ func pickOperator(d *dataTreeNavigator, context Context, expressionNode *Express
 			return Context{}, fmt.Errorf("cannot pick indicies from type %v (%v)", node.Tag, candidate.GetNicePath())
 		}
 
-		results.PushBack(candidate.CreateReplacement(replacement))
+		results.PushBack(candidate.CreateReplacementWithDocWrappers(replacement))
 	}
 
 	return context.ChildContext(results), nil

pkg/yqlib/operator_pick_test.go

@@ -14,6 +14,15 @@ var pickOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::myMap: {hamster: squeek, cat: meow}\n",
 		},
 	},
+	{
+		description: "Pick keys from map with comments",
+		skipDoc:     true,
+		document:    "# abc\nmyMap: {cat: meow, dog: bark, thing: hamster, hamster: squeek}\n# xyz\n",
+		expression:  `.myMap |= pick(["hamster", "cat", "goat"])`,
+		expected: []string{
+			"D0, P[], (doc)::# abc\nmyMap: {hamster: squeek, cat: meow}\n# xyz\n",
+		},
+	},
 	{
 		description:    "Pick indices from array",
 		subdescription: "Note that the order of the indexes matches the pick order and non existent indexes are skipped.",
@@ -23,6 +32,15 @@ var pickOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[lion, cat]\n",
 		},
 	},
+	{
+		description: "Pick indices from array with comments",
+		skipDoc:     true,
+		document:    "# abc\n[cat, leopard, lion]\n# xyz",
+		expression:  `pick([2, 0, 734, -5])`,
+		expected: []string{
+			"D0, P[], (!!seq)::# abc\n[lion, cat]\n# xyz\n",
+		},
+	},
 }
 
 func TestPickOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_reverse.go

@@ -25,7 +25,7 @@ func reverseOperator(d *dataTreeNavigator, context Context, expressionNode *Expr
 		for i, originalNode := range candidateNode.Content {
 			reverseList.Content[len(candidateNode.Content)-i-1] = originalNode
 		}
-		results.PushBack(candidate.CreateReplacement(reverseList))
+		results.PushBack(candidate.CreateReplacementWithDocWrappers(reverseList))
 
 	}
 

pkg/yqlib/operator_reverse_test.go

@@ -44,6 +44,15 @@ var reverseOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[{a: cat}, {a: banana}, {a: apple}]\n",
 		},
 	},
+	{
+		description: "Sort descending by string field, with comments",
+		skipDoc:     true,
+		document:    "# abc\n[{a: banana},{a: cat},{a: apple}]\n# xyz",
+		expression:  `sort_by(.a) | reverse`,
+		expected: []string{
+			"D0, P[], (!!seq)::# abc\n[{a: cat}, {a: banana}, {a: apple}]\n# xyz\n",
+		},
+	},
 }
 
 func TestReverseOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_sort.go

@@ -64,7 +64,7 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 		for i, sortedNode := range sortableArray {
 			sortedList.Content[i] = sortedNode.Node
 		}
-		results.PushBack(candidate.CreateReplacement(sortedList))
+		results.PushBack(candidate.CreateReplacementWithDocWrappers(sortedList))
 	}
 	return context.ChildContext(results), nil
 }

pkg/yqlib/operator_unique.go

@@ -56,7 +56,7 @@ func uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionN
 			resultNode.Content = append(resultNode.Content, el.Value.(*yaml.Node))
 		}
 
-		results.PushBack(candidate.CreateReplacement(resultNode))
+		results.PushBack(candidate.CreateReplacementWithDocWrappers(resultNode))
 	}
 
 	return context.ChildContext(results), nil

pkg/yqlib/operator_unique_test.go

@@ -56,6 +56,14 @@ var uniqueOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::- {name: harry, pet: cat}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "# abc\n[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]\n# xyz",
+		expression: `unique_by(.name)`,
+		expected: []string{
+			"D0, P[], (!!seq)::# abc\n- {name: harry, pet: cat}\n- {pet: fish}\n# xyz\n",
+		},
+	},
 }
 
 func TestUniqueOperatorScenarios(t *testing.T) {

pkg/yqlib/xml_test.go

@@ -153,6 +153,24 @@ var expectedXMLWithComments = `<!-- above_cat inline_cat --><cat><!-- above_arra
 </cat><!-- below_cat -->
 `
 
+var inputXMLWithNamespacedAttr = `
+<?xml version="1.0"?>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
+</map>
+`
+
+var expectedYAMLWithNamespacedAttr = `map:
+  +xmlns: some-namespace
+  +xmlns:xsi: some-instance
+  +some-instance:schemaLocation: some-url
+`
+
+var expectedYAMLWithRawNamespacedAttr = `map:
+  +xmlns: some-namespace
+  +xmlns:xsi: some-instance
+  +xsi:schemaLocation: some-url
+`
+
 var xmlWithCustomDtd = `
 <?xml version="1.0"?>
 <!DOCTYPE root [
@@ -254,6 +272,20 @@ var xmlScenarios = []formatScenario{
 		expected:     expectedDecodeYamlWithArray,
 		scenarioType: "decode",
 	},
+	{
+		description:  "Parse xml: keep attribute namespace",
+		skipDoc:      false,
+		input:        inputXMLWithNamespacedAttr,
+		expected:     expectedYAMLWithNamespacedAttr,
+		scenarioType: "decode-keep-ns",
+	},
+	{
+		description:  "Parse xml: keep raw attribute namespace",
+		skipDoc:      false,
+		input:        inputXMLWithNamespacedAttr,
+		expected:     expectedYAMLWithRawNamespacedAttr,
+		scenarioType: "decode-raw-token",
+	},
 	{
 		description:  "Encode xml: simple",
 		input:        "cat: purrs",
@@ -303,12 +335,17 @@ var xmlScenarios = []formatScenario{
 }
 
 func testXMLScenario(t *testing.T, s formatScenario) {
-	if s.scenarioType == "encode" {
+	switch s.scenarioType {
+	case "encode":
 		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, "+", "+content")), s.description)
-	} else if s.scenarioType == "roundtrip" {
+	case "roundtrip":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false), NewXMLEncoder(2, "+", "+content")), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false), NewXMLEncoder(2, "+", "+content")), s.description)
-	} else {
+	case "decode-keep-ns":
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false), NewYamlEncoder(4, false, true, true)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, false), NewYamlEncoder(2, false, true, true)), s.description)
+	case "decode-raw-token":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, true), NewYamlEncoder(2, false, true, true)), s.description)
+	default:
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false), NewYamlEncoder(4, false, true, true)), s.description)
 	}
 }
 
@@ -318,11 +355,16 @@ func documentXMLScenario(t *testing.T, w *bufio.Writer, i interface{}) {
 	if s.skipDoc {
 		return
 	}
-	if s.scenarioType == "encode" {
+	switch s.scenarioType {
+	case "encode":
 		documentXMLEncodeScenario(w, s)
-	} else if s.scenarioType == "roundtrip" {
+	case "roundtrip":
 		documentXMLRoundTripScenario(w, s)
-	} else {
+	case "decode-keep-ns":
+		documentXMLDecodeKeepNsScenario(w, s)
+	case "decode-raw-token":
+		documentXMLDecodeKeepNsRawTokenScenario(w, s)
+	default:
 		documentXMLDecodeScenario(w, s)
 	}
 
@@ -347,7 +389,49 @@ func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=xml '%v' sample.xml\n```\n", expression))
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false), NewYamlEncoder(2, false, true, true))))
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false), NewYamlEncoder(2, false, true, true))))
+}
+
+func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, "Given a sample.xml file of:\n")
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
+
+	writeOrPanic(w, "then\n")
+	writeOrPanic(w, "```bash\nyq -p=xml -o=xml --xml-keep-namespace '.' sample.xml\n```\n")
+	writeOrPanic(w, "will output\n")
+
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, false), NewXMLEncoder(2, "+", "+content"))))
+
+	writeOrPanic(w, "instead of\n")
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false), NewXMLEncoder(2, "+", "+content"))))
+}
+
+func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, "Given a sample.xml file of:\n")
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
+
+	writeOrPanic(w, "then\n")
+	writeOrPanic(w, "```bash\nyq -p=xml -o=xml --xml-keep-namespace --xml-raw-token '.' sample.xml\n```\n")
+	writeOrPanic(w, "will output\n")
+
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, true), NewXMLEncoder(2, "+", "+content"))))
+
+	writeOrPanic(w, "instead of\n")
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false), NewXMLEncoder(2, "+", "+content"))))
 }
 
 func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
@@ -383,7 +467,7 @@ func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, "```bash\nyq -p=xml -o=xml '.' sample.xml\n```\n")
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false), NewXMLEncoder(2, "+", "+content"))))
+	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false), NewXMLEncoder(2, "+", "+content"))))
 }
 
 func TestXMLScenarios(t *testing.T) {