Add tomlq. Drop Python 2.7 support.

.github/workflows/pythonpackage.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       max-parallel: 8
       matrix:
-        python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v1

README.rst

@@ -106,6 +106,12 @@ XML support
 the ``xq --xml-output``/``xq -x`` option. Multiple XML documents can be passed in separate files/streams as
 ``xq a.xml b.xml``. Entity expansion and DTD resolution is disabled to avoid XML parsing vulnerabilities.
 
+TOML support
+------------
+``yq`` supports `TOML <https://toml.io/>`_ as well. The ``yq`` package installs an executable, ``tomlq``, which uses the
+`toml library <https://github.com/uiri/toml>`_ to transcode TOML to JSON, then pipes it to ``jq``. Roundtrip transcoding
+is available with the ``tomlq --toml-output``/``tomlq -t`` option.
+
 .. admonition:: Compatibility note
 
  This package's release series available on PyPI begins with version 2.0.0. Versions of ``yq`` prior to 2.0.0 are

setup.py

@@ -17,6 +17,7 @@
         "setuptools",
         "PyYAML >= 3.11",
         "xmltodict >= 0.11.0",
+        "toml >= 0.10.0",
         "argcomplete >= 1.8.1"
     ],
     tests_require=tests_require,
@@ -28,7 +29,8 @@
     entry_points={
         'console_scripts': [
             'yq=yq:cli',
-            'xq=yq:xq_cli'
+            'xq=yq:xq_cli',
+            'tomlq=yq:tq_cli'
         ],
     },
     test_suite="test",
@@ -38,11 +40,11 @@
         "Operating System :: MacOS :: MacOS X",
         "Operating System :: POSIX",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Topic :: Software Development :: Libraries :: Python Modules"
     ]
 )

test/test.py

@@ -1,14 +1,10 @@
 #!/usr/bin/env python
-# coding: utf-8
 
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-import os, sys, unittest, tempfile, json, io, platform, subprocess, yaml
+import os, sys, unittest, tempfile, io, platform, subprocess, yaml
 
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
 from yq import yq, cli  # noqa
 
-USING_PYTHON2 = True if sys.version_info < (3, 0) else False
 USING_PYPY = True if platform.python_implementation() == "PyPy" else False
 
 yaml_with_tags = """
@@ -36,19 +32,16 @@ class TestYq(unittest.TestCase):
     def run_yq(self, input_data, args, expect_exit_codes={os.EX_OK}, input_format="yaml"):
         stdin, stdout = sys.stdin, sys.stdout
         try:
-            str_type = unicode if USING_PYTHON2 else str
-            if isinstance(input_data, str_type):
+            if isinstance(input_data, str):
                 sys.stdin = io.StringIO(input_data)
             else:
                 sys.stdin = input_data
-            sys.stdout = io.BytesIO() if USING_PYTHON2 else io.StringIO()
+            sys.stdout = io.StringIO()
             cli(args, input_format=input_format)
         except SystemExit as e:
             self.assertIn(e.code, expect_exit_codes)
         finally:
             result = sys.stdout.getvalue()
-            if USING_PYTHON2:
-                result = result.decode("utf-8")
             sys.stdin, sys.stdout = stdin, stdout
         return result
 
@@ -100,8 +93,7 @@ def test_yq_arg_passthrough(self):
         self.assertEqual(self.run_yq("{}", ["--arg", "foo", "bar", "--arg", "x", "y", "--indent", "4", "."]), "")
         self.assertEqual(self.run_yq("{}", ["--arg", "foo", "bar", "--arg", "x", "y", "-y", "--indent", "4", ".x=$x"]),
                          "x: y\n")
-        err = "yq: Error running jq: {}Error: [Errno 32] Broken pipe{}".format("IO" if USING_PYTHON2 else "BrokenPipe",
-                                                                               ": '<fdopen>'." if USING_PYPY else ".")
+        err = "yq: Error running jq: BrokenPipeError: [Errno 32] Broken pipe" + (": '<fdopen>'." if USING_PYPY else ".")
         self.run_yq("{}", ["--indent", "9", "."], expect_exit_codes={err, 2})
 
         with tempfile.NamedTemporaryFile() as tf, tempfile.TemporaryFile() as tf2:
@@ -234,6 +226,10 @@ def test_xq_dtd(self):
                 '<?xml version="1.0" encoding="utf-8"?>\n<g>\n  <b c="d">e</b>\n  <b>f</b>\n</g>\n'
             )
 
+    def test_tomlq(self):
+        self.assertEqual(self.run_yq("[foo]\nbar = 1", ["."], input_format="toml"), "")
+        self.assertEqual(self.run_yq("[foo]\nbar = 1", ["-t", ".foo"], input_format="toml"), "bar = 1\n")
+
     @unittest.skipIf(sys.version_info < (3, 5),
                      "argparse option abbreviation interferes with opt passthrough, can't be disabled until Python 3.5")
     def test_abbrev_opt_collisions(self):

yq/__init__.py

@@ -7,15 +7,12 @@
 
 # PYTHON_ARGCOMPLETE_OK
 
-from __future__ import absolute_import, division, print_function, unicode_literals
-
 import sys, argparse, subprocess, json
 from collections import OrderedDict
 from datetime import datetime, date, time
 
 import yaml, argcomplete
 
-from .compat import USING_PYTHON2, open
 from .parser import get_parser, jq_arg_spec
 from .loader import get_loader
 from .dumper import get_dumper
@@ -37,7 +34,7 @@ def xq_cli():
     cli(input_format="xml", program_name="xq")
 
 def tq_cli():
-    cli(input_format="toml", program_name="tq")
+    cli(input_format="toml", program_name="tomlq")
 
 class DeferredOutputStream:
     def __init__(self, name, mode="w"):
@@ -115,8 +112,6 @@ def cli(args=None, input_format="yaml", program_name="yq"):
 
     yq_args = dict(input_format=input_format, program_name=program_name, jq_args=jq_args, **vars(args))
     if in_place:
-        if USING_PYTHON2:
-            sys.exit("{}: -i/--in-place is not compatible with Python 2".format(program_name))
         if args.output_format not in {"yaml", "annotated_yaml"}:
             sys.exit("{}: -i/--in-place can only be used with -y/-Y".format(program_name))
         input_streams = yq_args.pop("input_streams")
@@ -207,14 +202,7 @@ def yq(input_streams=None, output_stream=None, input_format="yaml", output_forma
                     if not isinstance(doc, OrderedDict):
                         msg = "{}: Error converting JSON to TOML: cannot represent non-object types at top level."
                         exit_func(msg.format(program_name))
-
-                    if USING_PYTHON2:
-                        # For Python 2, dump the string and encode it into bytes.
-                        output = toml.dumps(doc)
-                        output_stream.write(output.encode("utf-8"))
-                    else:
-                        # For Python 3, write the unicode to the buffer directly.
-                        toml.dump(doc, output_stream)
+                    toml.dump(doc, output_stream)
         else:
             if input_format == "yaml":
                 loader = get_loader(use_annotations=False)

yq/dumper.py

@@ -1,11 +1,8 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
-
 import re
 from collections import OrderedDict
 
 import yaml
 
-from .compat import str
 from .loader import hash_key
 
 class OrderedIndentlessDumper(yaml.SafeDumper):

yq/loader.py

@@ -1,12 +1,8 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
-
 import yaml
 from base64 import b64encode
 from collections import OrderedDict
 from hashlib import sha224
 
-from .compat import str
-
 def hash_key(key):
     return b64encode(sha224(key.encode() if isinstance(key, str) else key).digest()).decode()
 

yq/parser.py

@@ -11,7 +11,7 @@
 class Parser(argparse.ArgumentParser):
     def print_help(self):
         yq_help = argparse.ArgumentParser.format_help(self).splitlines()
-        print("\n".join(["usage: yq [options] <jq filter> [YAML file...]"] + yq_help[1:] + [""]))
+        print("\n".join(["usage: {} [options] <jq filter> [input file...]".format(self.prog)] + yq_help[2:] + [""]))
         sys.stdout.flush()
         try:
             subprocess.check_call(["jq", "--help"])
@@ -43,7 +43,7 @@ def get_parser(program_name, description):
         xml_root_help = "When transcoding back to XML, envelope the output in an element with this name"
         xml_force_list_help = ("Tag name to pass to force_list parameter of xmltodict.parse(). "
                                "Can be used multiple times.")
-    elif program_name == "tq":
+    elif program_name == "tomlq":
         current_language = "TOML"
         toml_output_help = "Transcode jq JSON output back into TOML and emit it"
     else: