diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml
index 19f8dd9..8db58e8 100644
--- a/.github/workflows/pythonpackage.yml
+++ b/.github/workflows/pythonpackage.yml
@@ -8,7 +8,7 @@ jobs:
strategy:
max-parallel: 8
matrix:
- python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
+ python-version: [3.5, 3.6, 3.7, 3.8, 3.9]
steps:
- uses: actions/checkout@v1
diff --git a/README.rst b/README.rst
index 8bd0d7c..1ab3614 100644
--- a/README.rst
+++ b/README.rst
@@ -106,6 +106,12 @@ XML support
the ``xq --xml-output``/``xq -x`` option. Multiple XML documents can be passed in separate files/streams as
``xq a.xml b.xml``. Entity expansion and DTD resolution is disabled to avoid XML parsing vulnerabilities.
+TOML support
+------------
+``yq`` supports `TOML `_ as well. The ``yq`` package installs an executable, ``tomlq``, which uses the
+`toml library `_ to transcode TOML to JSON, then pipes it to ``jq``. Roundtrip transcoding
+is available with the ``tomlq --toml-output``/``tomlq -t`` option.
+
.. admonition:: Compatibility note
This package's release series available on PyPI begins with version 2.0.0. Versions of ``yq`` prior to 2.0.0 are
diff --git a/setup.py b/setup.py
index 46eb9b7..3b73d28 100755
--- a/setup.py
+++ b/setup.py
@@ -17,6 +17,7 @@
"setuptools",
"PyYAML >= 3.11",
"xmltodict >= 0.11.0",
+ "toml >= 0.10.0",
"argcomplete >= 1.8.1"
],
tests_require=tests_require,
@@ -28,7 +29,8 @@
entry_points={
'console_scripts': [
'yq=yq:cli',
- 'xq=yq:xq_cli'
+ 'xq=yq:xq_cli',
+ 'tomlq=yq:tq_cli'
],
},
test_suite="test",
@@ -38,11 +40,11 @@
"Operating System :: MacOS :: MacOS X",
"Operating System :: POSIX",
"Programming Language :: Python",
- "Programming Language :: Python :: 2.7",
"Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
"Topic :: Software Development :: Libraries :: Python Modules"
]
)
diff --git a/test/test.py b/test/test.py
index a81f41b..80c4441 100755
--- a/test/test.py
+++ b/test/test.py
@@ -1,14 +1,10 @@
#!/usr/bin/env python
-# coding: utf-8
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-import os, sys, unittest, tempfile, json, io, platform, subprocess, yaml
+import os, sys, unittest, tempfile, io, platform, subprocess, yaml
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from yq import yq, cli # noqa
-USING_PYTHON2 = True if sys.version_info < (3, 0) else False
USING_PYPY = True if platform.python_implementation() == "PyPy" else False
yaml_with_tags = """
@@ -36,19 +32,16 @@ class TestYq(unittest.TestCase):
def run_yq(self, input_data, args, expect_exit_codes={os.EX_OK}, input_format="yaml"):
stdin, stdout = sys.stdin, sys.stdout
try:
- str_type = unicode if USING_PYTHON2 else str
- if isinstance(input_data, str_type):
+ if isinstance(input_data, str):
sys.stdin = io.StringIO(input_data)
else:
sys.stdin = input_data
- sys.stdout = io.BytesIO() if USING_PYTHON2 else io.StringIO()
+ sys.stdout = io.StringIO()
cli(args, input_format=input_format)
except SystemExit as e:
self.assertIn(e.code, expect_exit_codes)
finally:
result = sys.stdout.getvalue()
- if USING_PYTHON2:
- result = result.decode("utf-8")
sys.stdin, sys.stdout = stdin, stdout
return result
@@ -100,8 +93,7 @@ def test_yq_arg_passthrough(self):
self.assertEqual(self.run_yq("{}", ["--arg", "foo", "bar", "--arg", "x", "y", "--indent", "4", "."]), "")
self.assertEqual(self.run_yq("{}", ["--arg", "foo", "bar", "--arg", "x", "y", "-y", "--indent", "4", ".x=$x"]),
"x: y\n")
- err = "yq: Error running jq: {}Error: [Errno 32] Broken pipe{}".format("IO" if USING_PYTHON2 else "BrokenPipe",
- ": ''." if USING_PYPY else ".")
+ err = "yq: Error running jq: BrokenPipeError: [Errno 32] Broken pipe" + (": ''." if USING_PYPY else ".")
self.run_yq("{}", ["--indent", "9", "."], expect_exit_codes={err, 2})
with tempfile.NamedTemporaryFile() as tf, tempfile.TemporaryFile() as tf2:
@@ -234,6 +226,10 @@ def test_xq_dtd(self):
'\n\n e\n f\n\n'
)
+ def test_tomlq(self):
+ self.assertEqual(self.run_yq("[foo]\nbar = 1", ["."], input_format="toml"), "")
+ self.assertEqual(self.run_yq("[foo]\nbar = 1", ["-t", ".foo"], input_format="toml"), "bar = 1\n")
+
@unittest.skipIf(sys.version_info < (3, 5),
"argparse option abbreviation interferes with opt passthrough, can't be disabled until Python 3.5")
def test_abbrev_opt_collisions(self):
diff --git a/yq/__init__.py b/yq/__init__.py
index afeb42c..a83500f 100755
--- a/yq/__init__.py
+++ b/yq/__init__.py
@@ -7,15 +7,12 @@
# PYTHON_ARGCOMPLETE_OK
-from __future__ import absolute_import, division, print_function, unicode_literals
-
import sys, argparse, subprocess, json
from collections import OrderedDict
from datetime import datetime, date, time
import yaml, argcomplete
-from .compat import USING_PYTHON2, open
from .parser import get_parser, jq_arg_spec
from .loader import get_loader
from .dumper import get_dumper
@@ -37,7 +34,7 @@ def xq_cli():
cli(input_format="xml", program_name="xq")
def tq_cli():
- cli(input_format="toml", program_name="tq")
+ cli(input_format="toml", program_name="tomlq")
class DeferredOutputStream:
def __init__(self, name, mode="w"):
@@ -115,8 +112,6 @@ def cli(args=None, input_format="yaml", program_name="yq"):
yq_args = dict(input_format=input_format, program_name=program_name, jq_args=jq_args, **vars(args))
if in_place:
- if USING_PYTHON2:
- sys.exit("{}: -i/--in-place is not compatible with Python 2".format(program_name))
if args.output_format not in {"yaml", "annotated_yaml"}:
sys.exit("{}: -i/--in-place can only be used with -y/-Y".format(program_name))
input_streams = yq_args.pop("input_streams")
@@ -207,14 +202,7 @@ def yq(input_streams=None, output_stream=None, input_format="yaml", output_forma
if not isinstance(doc, OrderedDict):
msg = "{}: Error converting JSON to TOML: cannot represent non-object types at top level."
exit_func(msg.format(program_name))
-
- if USING_PYTHON2:
- # For Python 2, dump the string and encode it into bytes.
- output = toml.dumps(doc)
- output_stream.write(output.encode("utf-8"))
- else:
- # For Python 3, write the unicode to the buffer directly.
- toml.dump(doc, output_stream)
+ toml.dump(doc, output_stream)
else:
if input_format == "yaml":
loader = get_loader(use_annotations=False)
diff --git a/yq/compat.py b/yq/compat.py
deleted file mode 100644
index ad7360b..0000000
--- a/yq/compat.py
+++ /dev/null
@@ -1,12 +0,0 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-import sys, io
-
-USING_PYTHON2 = True if sys.version_info < (3, 0) else False
-
-if USING_PYTHON2:
- str = unicode # noqa
- open = io.open
-else:
- str = str
- open = open
diff --git a/yq/dumper.py b/yq/dumper.py
index 575436d..294e4a7 100644
--- a/yq/dumper.py
+++ b/yq/dumper.py
@@ -1,11 +1,8 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
-
import re
from collections import OrderedDict
import yaml
-from .compat import str
from .loader import hash_key
class OrderedIndentlessDumper(yaml.SafeDumper):
diff --git a/yq/loader.py b/yq/loader.py
index 411ed86..fde955f 100644
--- a/yq/loader.py
+++ b/yq/loader.py
@@ -1,12 +1,8 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
-
import yaml
from base64 import b64encode
from collections import OrderedDict
from hashlib import sha224
-from .compat import str
-
def hash_key(key):
return b64encode(sha224(key.encode() if isinstance(key, str) else key).digest()).decode()
diff --git a/yq/parser.py b/yq/parser.py
index fa5828b..46c8aa3 100644
--- a/yq/parser.py
+++ b/yq/parser.py
@@ -11,7 +11,7 @@
class Parser(argparse.ArgumentParser):
def print_help(self):
yq_help = argparse.ArgumentParser.format_help(self).splitlines()
- print("\n".join(["usage: yq [options] [YAML file...]"] + yq_help[1:] + [""]))
+ print("\n".join(["usage: {} [options] [input file...]".format(self.prog)] + yq_help[2:] + [""]))
sys.stdout.flush()
try:
subprocess.check_call(["jq", "--help"])
@@ -43,7 +43,7 @@ def get_parser(program_name, description):
xml_root_help = "When transcoding back to XML, envelope the output in an element with this name"
xml_force_list_help = ("Tag name to pass to force_list parameter of xmltodict.parse(). "
"Can be used multiple times.")
- elif program_name == "tq":
+ elif program_name == "tomlq":
current_language = "TOML"
toml_output_help = "Transcode jq JSON output back into TOML and emit it"
else: