Merge 53b57f8abb into a68c3aa69e

.github/workflows/ci.yaml

@@ -33,16 +33,21 @@ jobs:
 
   test:
     name: Tests
-    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
+        os: ['macos', 'ubuntu', 'windows']
         python-version:
           - '3.7'
           - '3.8'
           - '3.9'
           - '3.10'
           - '3.11'
+    runs-on: ${{ matrix.os }}-latest
+    defaults:
+      run:
+        shell: bash -e {0}
+
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -50,12 +55,20 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+      - run: pip install .
       - name: Append GitHub Actions system path
+        if: ${{ matrix.os == 'ubuntu' }}
         run: echo "$HOME/.local/bin" >> $GITHUB_PATH
       - run: pip install coverage
-      - run: pip install .
+        if: ${{ matrix.os == 'ubuntu' }}
       # https://github.com/AndreMiras/coveralls-python-action/issues/18
       - run: echo -e "[run]\nrelative_files = True" > .coveragerc
+        if: ${{ matrix.os == 'ubuntu' }}
       - run: coverage run -m unittest discover
+        if: ${{ matrix.os == 'ubuntu' }}
       - name: Coveralls
+        if: ${{ matrix.os == 'ubuntu' }}
         uses: AndreMiras/coveralls-python-action@develop
+      - name: Unittests only
+        if: ${{ matrix.os != 'ubuntu' }}
+        run: python -m unittest

tests/common.py

@@ -65,9 +65,12 @@ def build_temp_workspace(files):
         if type(content) is list:
             os.mkdir(path)
         else:
-            mode = 'wb' if isinstance(content, bytes) else 'w'
+            if isinstance(content, bytes):
-            with open(path, mode) as f:
+                with open(path, 'wb') as f:
-                f.write(content)
+                    f.write(content)
+            else:
+                with open(path, 'w', newline='') as f:
+                    f.write(content)
 
     return tempdir
 
@@ -84,3 +87,18 @@ def temp_workspace(files):
     finally:
         os.chdir(backup_wd)
         shutil.rmtree(wd)
+
+
+@contextlib.contextmanager
+def CompatNamedTemporaryFile(*args, **kwargs):
+    try:
+        assert 'delete' not in kwargs, "not applicable"
+        f = tempfile.NamedTemporaryFile(*args, **kwargs, delete=False)
+        yield f
+    finally:
+        f.close()
+        os.unlink(f.name)
+
+
+def rsep(s: str) -> str:
+    return s.replace('/', os.sep)

tests/rules/test_key_ordering.py

@@ -116,6 +116,8 @@ class KeyOrderingTestCase(RuleTestCase):
         self.addCleanup(locale.setlocale, locale.LC_ALL, (None, None))
         try:
             locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
+            if not locale.strcoll('t', 'T') < 0:  # pragma: no cover
+                self.skipTest("Not 't' < 'T' for locale en_US.UTF-8")
         except locale.Error:  # pragma: no cover
             self.skipTest('locale en_US.UTF-8 not available')
         conf = ('key-ordering: enable')

tests/test_cli.py

@@ -14,21 +14,25 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from io import StringIO
-import fcntl
 import locale
 import os
-import pty
 import shutil
 import sys
 import tempfile
 import unittest
 
-from tests.common import build_temp_workspace, temp_workspace
+from tests.common import (build_temp_workspace, temp_workspace,
+                          CompatNamedTemporaryFile, rsep)
 
 from yamllint import cli
 from yamllint import config
 
 
+if not sys.platform.startswith('win'):
+    import pty
+    import fcntl
+
+
 class RunContext:
     """Context manager for ``cli.run()`` to capture exit code and streams."""
 
@@ -127,9 +131,10 @@ class CommandLineTestCase(unittest.TestCase):
              os.path.join(self.wd, 'dos.yml'),
              os.path.join(self.wd, 'empty.yml'),
              os.path.join(self.wd, 'en.yaml'),
-             os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'),
+             os.path.join(self.wd,
-             os.path.join(self.wd, 'sub/directory.yaml/empty.yml'),
+                          rsep('s/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml')),
-             os.path.join(self.wd, 'sub/ok.yaml'),
+             os.path.join(self.wd, rsep('sub/directory.yaml/empty.yml')),
+             os.path.join(self.wd, rsep('sub/ok.yaml')),
              os.path.join(self.wd, 'warn.yaml')],
         )
 
@@ -145,7 +150,8 @@ class CommandLineTestCase(unittest.TestCase):
         self.assertEqual(
             sorted(cli.find_files_recursively(items, conf)),
             [os.path.join(self.wd, 'empty.yml'),
-             os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml')],
+             os.path.join(self.wd,
+                          rsep('s/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'))],
         )
 
         items = [os.path.join(self.wd, 'sub'),
@@ -153,8 +159,8 @@ class CommandLineTestCase(unittest.TestCase):
         self.assertEqual(
             sorted(cli.find_files_recursively(items, conf)),
             [os.path.join(self.wd, '/etc/another/file'),
-             os.path.join(self.wd, 'sub/directory.yaml/empty.yml'),
+             os.path.join(self.wd, rsep('sub/directory.yaml/empty.yml')),
-             os.path.join(self.wd, 'sub/ok.yaml')],
+             os.path.join(self.wd, rsep('sub/ok.yaml'))],
         )
 
         conf = config.YamlLintConfig('extends: default\n'
@@ -165,8 +171,9 @@ class CommandLineTestCase(unittest.TestCase):
             [os.path.join(self.wd, 'a.yaml'),
              os.path.join(self.wd, 'c.yaml'),
              os.path.join(self.wd, 'en.yaml'),
-             os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'),
+             os.path.join(self.wd,
-             os.path.join(self.wd, 'sub/ok.yaml'),
+                          rsep('s/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml')),
+             os.path.join(self.wd, rsep('sub/ok.yaml')),
              os.path.join(self.wd, 'warn.yaml')]
         )
 
@@ -177,7 +184,7 @@ class CommandLineTestCase(unittest.TestCase):
             sorted(cli.find_files_recursively([self.wd], conf)),
             [os.path.join(self.wd, 'dos.yml'),
              os.path.join(self.wd, 'empty.yml'),
-             os.path.join(self.wd, 'sub/directory.yaml/empty.yml')]
+             os.path.join(self.wd, rsep('sub/directory.yaml/empty.yml'))]
         )
 
         conf = config.YamlLintConfig('extends: default\n'
@@ -199,11 +206,12 @@ class CommandLineTestCase(unittest.TestCase):
              os.path.join(self.wd, 'empty.yml'),
              os.path.join(self.wd, 'en.yaml'),
              os.path.join(self.wd, 'no-yaml.json'),
-             os.path.join(self.wd, 'non-ascii/éçäγλνπ¥/utf-8'),
+             os.path.join(self.wd, rsep('non-ascii/éçäγλνπ¥/utf-8')),
-             os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'),
+             os.path.join(self.wd,
-             os.path.join(self.wd, 'sub/directory.yaml/empty.yml'),
+                          rsep('s/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml')),
-             os.path.join(self.wd, 'sub/directory.yaml/not-yaml.txt'),
+             os.path.join(self.wd, rsep('sub/directory.yaml/empty.yml')),
-             os.path.join(self.wd, 'sub/ok.yaml'),
+             os.path.join(self.wd, rsep('sub/directory.yaml/not-yaml.txt')),
+             os.path.join(self.wd, rsep('sub/ok.yaml')),
              os.path.join(self.wd, 'warn.yaml')]
         )
 
@@ -220,11 +228,12 @@ class CommandLineTestCase(unittest.TestCase):
              os.path.join(self.wd, 'empty.yml'),
              os.path.join(self.wd, 'en.yaml'),
              os.path.join(self.wd, 'no-yaml.json'),
-             os.path.join(self.wd, 'non-ascii/éçäγλνπ¥/utf-8'),
+             os.path.join(self.wd, rsep('non-ascii/éçäγλνπ¥/utf-8')),
-             os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'),
+             os.path.join(self.wd,
-             os.path.join(self.wd, 'sub/directory.yaml/empty.yml'),
+                          rsep('s/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml')),
-             os.path.join(self.wd, 'sub/directory.yaml/not-yaml.txt'),
+             os.path.join(self.wd, rsep('sub/directory.yaml/empty.yml')),
-             os.path.join(self.wd, 'sub/ok.yaml'),
+             os.path.join(self.wd, rsep('sub/directory.yaml/not-yaml.txt')),
+             os.path.join(self.wd, rsep('sub/ok.yaml')),
              os.path.join(self.wd, 'warn.yaml')]
         )
 
@@ -234,7 +243,7 @@ class CommandLineTestCase(unittest.TestCase):
                                      '  - \'**/utf-8\'\n')
         self.assertEqual(
             sorted(cli.find_files_recursively([self.wd], conf)),
-            [os.path.join(self.wd, 'non-ascii/éçäγλνπ¥/utf-8')]
+            [os.path.join(self.wd, rsep('non-ascii/éçäγλνπ¥/utf-8'))]
         )
 
     def test_run_with_bad_arguments(self):
@@ -306,6 +315,8 @@ class CommandLineTestCase(unittest.TestCase):
             cli.run(('-c', f.name, os.path.join(self.wd, 'a.yaml')))
         self.assertEqual(ctx.returncode, 1)
 
+    @unittest.skipIf(sys.platform.startswith('win'),
+                     'TODO Windows override HOME unimplemented')
     @unittest.skipIf(os.environ.get('GITHUB_RUN_ID'), '$HOME not overridable')
     def test_run_with_user_global_config_file(self):
         home = os.path.join(self.wd, 'fake-home')
@@ -346,7 +357,7 @@ class CommandLineTestCase(unittest.TestCase):
     def test_run_with_user_yamllint_config_file_in_env(self):
         self.addCleanup(os.environ.__delitem__, 'YAMLLINT_CONFIG_FILE')
 
-        with tempfile.NamedTemporaryFile('w') as f:
+        with CompatNamedTemporaryFile('w') as f:
             os.environ['YAMLLINT_CONFIG_FILE'] = f.name
             f.write('rules: {trailing-spaces: disable}')
             f.flush()
@@ -354,7 +365,7 @@ class CommandLineTestCase(unittest.TestCase):
                 cli.run((os.path.join(self.wd, 'a.yaml'), ))
             self.assertEqual(ctx.returncode, 0)
 
-        with tempfile.NamedTemporaryFile('w') as f:
+        with CompatNamedTemporaryFile('w') as f:
             os.environ['YAMLLINT_CONFIG_FILE'] = f.name
             f.write('rules: {trailing-spaces: enable}')
             f.flush()
@@ -368,6 +379,8 @@ class CommandLineTestCase(unittest.TestCase):
         # as the first two runs don't use setlocale()
         try:
             locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
+            if not locale.strcoll('a', 'A') < 0:  # pragma: no cover
+                self.skipTest("Not 'a' < 'A' for locale en_US.UTF-8")
         except locale.Error:  # pragma: no cover
             self.skipTest('locale en_US.UTF-8 not available')
         locale.setlocale(locale.LC_ALL, (None, None))
@@ -477,7 +490,7 @@ class CommandLineTestCase(unittest.TestCase):
         self.assertEqual((ctx.returncode, ctx.stderr), (1, ''))
         self.assertEqual(ctx.stdout, (
             '%s:3:1: [error] duplication of key "key" in mapping '
-            '(key-duplicates)\n') % path)
+            '(key-duplicates)\n') % rsep(path))
 
     def test_run_piped_output_nocolor(self):
         path = os.path.join(self.wd, 'a.yaml')
@@ -492,6 +505,8 @@ class CommandLineTestCase(unittest.TestCase):
             '(new-line-at-end-of-file)\n'
             '\n' % path))
 
+    @unittest.skipIf(sys.platform.startswith('win'),
+                     'Windows pseudo-TTY unimplemented')
     def test_run_default_format_output_in_tty(self):
         path = os.path.join(self.wd, 'a.yaml')
 
@@ -689,9 +704,10 @@ class CommandLineTestCase(unittest.TestCase):
              os.path.join(self.wd, 'dos.yml'),
              os.path.join(self.wd, 'empty.yml'),
              os.path.join(self.wd, 'en.yaml'),
-             os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'),
+             os.path.join(self.wd,
-             os.path.join(self.wd, 'sub/directory.yaml/empty.yml'),
+                          rsep('s/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml')),
-             os.path.join(self.wd, 'sub/ok.yaml'),
+             os.path.join(self.wd, rsep('sub/directory.yaml/empty.yml')),
+             os.path.join(self.wd, rsep('sub/ok.yaml')),
              os.path.join(self.wd, 'warn.yaml')]
         )
 
@@ -705,9 +721,10 @@ class CommandLineTestCase(unittest.TestCase):
              os.path.join(self.wd, 'c.yaml'),
              os.path.join(self.wd, 'en.yaml'),
              os.path.join(self.wd, 'no-yaml.json'),
-             os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'),
+             os.path.join(self.wd,
-             os.path.join(self.wd, 'sub/directory.yaml/not-yaml.txt'),
+                          rsep('s/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml')),
-             os.path.join(self.wd, 'sub/ok.yaml'),
+             os.path.join(self.wd, rsep('sub/directory.yaml/not-yaml.txt')),
+             os.path.join(self.wd, rsep('sub/ok.yaml')),
              os.path.join(self.wd, 'warn.yaml')]
         )
 
@@ -724,9 +741,10 @@ class CommandLineConfigTestCase(unittest.TestCase):
                     with RunContext(self) as ctx:
                         cli.run(('-f', 'parsable', '.'))
 
-                self.assertEqual((ctx.returncode, ctx.stdout, ctx.stderr),
+                self.assertEqual(
-                                 (0, './a.yml:1:1: [warning] missing document '
+                    (ctx.returncode, ctx.stdout, ctx.stderr),
-                                     'start "---" (document-start)\n', ''))
+                    (0, rsep('./a.yml:1:1: [warning] missing document '
+                             'start "---" (document-start)\n'), ''))
 
                 with temp_workspace({**workspace, **{conf_file: conf}}):
                     with RunContext(self) as ctx:
@@ -747,10 +765,11 @@ class CommandLineConfigTestCase(unittest.TestCase):
                         os.chdir('a/b/c/d/e/f')
                         cli.run(('-f', 'parsable', '.'))
 
-                self.assertEqual((ctx.returncode, ctx.stdout, ctx.stderr),
+                self.assertEqual(
-                                 (0, './g/a.yml:1:1: [warning] missing '
+                    (ctx.returncode, ctx.stdout, ctx.stderr),
-                                     'document start "---" (document-start)\n',
+                    (0, rsep('./g/a.yml:1:1: [warning] missing '
-                                     ''))
+                             'document start "---" (document-start)\n'),
+                     ''))
 
                 with temp_workspace({**workspace, **{conf_file: conf}}):
                     with RunContext(self) as ctx:
@@ -783,15 +802,17 @@ class CommandLineConfigTestCase(unittest.TestCase):
                 os.chdir('a/b/c')
                 cli.run(('-f', 'parsable', '.'))
 
-        self.assertEqual((ctx.returncode, ctx.stdout, ctx.stderr),
+        self.assertEqual(
-                         (0, './3spaces.yml:2:4: [warning] wrong indentation: '
+            (ctx.returncode, ctx.stdout, ctx.stderr),
-                         'expected 4 but found 3 (indentation)\n', ''))
+            (0, rsep('./3spaces.yml:2:4: [warning] wrong indentation: '
+                     'expected 4 but found 3 (indentation)\n'), ''))
 
         with temp_workspace({**workspace, **{'a/b/.yamllint.yml': conf3}}):
             with RunContext(self) as ctx:
                 os.chdir('a/b/c')
                 cli.run(('-f', 'parsable', '.'))
 
-        self.assertEqual((ctx.returncode, ctx.stdout, ctx.stderr),
+        self.assertEqual(
-                         (0, './4spaces.yml:2:5: [warning] wrong indentation: '
+            (ctx.returncode, ctx.stdout, ctx.stderr),
-                         'expected 3 but found 4 (indentation)\n', ''))
+            (0, rsep('./4spaces.yml:2:5: [warning] wrong indentation: '
+                     'expected 3 but found 4 (indentation)\n'), ''))

tests/test_config.py

@@ -17,10 +17,9 @@ from io import StringIO
 import os
 import shutil
 import sys
-import tempfile
 import unittest
 
-from tests.common import build_temp_workspace
+from tests.common import build_temp_workspace, CompatNamedTemporaryFile, rsep
 
 from yamllint.config import YamlLintConfigError
 from yamllint import cli
@@ -245,7 +244,7 @@ class ExtendedConfigTestCase(unittest.TestCase):
         self.assertEqual(len(new.enabled_rules(None)), 2)
 
     def test_extend_on_file(self):
-        with tempfile.NamedTemporaryFile('w') as f:
+        with CompatNamedTemporaryFile('w') as f:
             f.write('rules:\n'
                     '  colons:\n'
                     '    max-spaces-before: 0\n'
@@ -264,7 +263,7 @@ class ExtendedConfigTestCase(unittest.TestCase):
         self.assertEqual(len(c.enabled_rules(None)), 2)
 
     def test_extend_remove_rule(self):
-        with tempfile.NamedTemporaryFile('w') as f:
+        with CompatNamedTemporaryFile('w') as f:
             f.write('rules:\n'
                     '  colons:\n'
                     '    max-spaces-before: 0\n'
@@ -283,7 +282,7 @@ class ExtendedConfigTestCase(unittest.TestCase):
         self.assertEqual(len(c.enabled_rules(None)), 1)
 
     def test_extend_edit_rule(self):
-        with tempfile.NamedTemporaryFile('w') as f:
+        with CompatNamedTemporaryFile('w') as f:
             f.write('rules:\n'
                     '  colons:\n'
                     '    max-spaces-before: 0\n'
@@ -305,7 +304,7 @@ class ExtendedConfigTestCase(unittest.TestCase):
         self.assertEqual(len(c.enabled_rules(None)), 2)
 
     def test_extend_reenable_rule(self):
-        with tempfile.NamedTemporaryFile('w') as f:
+        with CompatNamedTemporaryFile('w') as f:
             f.write('rules:\n'
                     '  colons:\n'
                     '    max-spaces-before: 0\n'
@@ -325,7 +324,7 @@ class ExtendedConfigTestCase(unittest.TestCase):
         self.assertEqual(len(c.enabled_rules(None)), 2)
 
     def test_extend_recursive_default_values(self):
-        with tempfile.NamedTemporaryFile('w') as f:
+        with CompatNamedTemporaryFile('w') as f:
             f.write('rules:\n'
                     '  braces:\n'
                     '    max-spaces-inside: 1248\n')
@@ -340,7 +339,7 @@ class ExtendedConfigTestCase(unittest.TestCase):
         self.assertEqual(c.rules['braces']['min-spaces-inside-empty'], 2357)
         self.assertEqual(c.rules['braces']['max-spaces-inside-empty'], -1)
 
-        with tempfile.NamedTemporaryFile('w') as f:
+        with CompatNamedTemporaryFile('w') as f:
             f.write('rules:\n'
                     '  colons:\n'
                     '    max-spaces-before: 1337\n')
@@ -352,8 +351,8 @@ class ExtendedConfigTestCase(unittest.TestCase):
         self.assertEqual(c.rules['colons']['max-spaces-before'], 1337)
         self.assertEqual(c.rules['colons']['max-spaces-after'], 1)
 
-        with tempfile.NamedTemporaryFile('w') as f1, \
+        with CompatNamedTemporaryFile('w') as f1, \
-                tempfile.NamedTemporaryFile('w') as f2:
+                CompatNamedTemporaryFile('w') as f2:
             f1.write('rules:\n'
                      '  colons:\n'
                      '    max-spaces-before: 1337\n')
@@ -370,7 +369,7 @@ class ExtendedConfigTestCase(unittest.TestCase):
         self.assertEqual(c.rules['colons']['max-spaces-after'], 1)
 
     def test_extended_ignore_str(self):
-        with tempfile.NamedTemporaryFile('w') as f:
+        with CompatNamedTemporaryFile('w') as f:
             f.write('ignore: |\n'
                     '  *.template.yaml\n')
             f.flush()
@@ -380,7 +379,7 @@ class ExtendedConfigTestCase(unittest.TestCase):
         self.assertEqual(c.ignore.match_file('test.yaml'), False)
 
     def test_extended_ignore_list(self):
-        with tempfile.NamedTemporaryFile('w') as f:
+        with CompatNamedTemporaryFile('w') as f:
             f.write('ignore:\n'
                     '  - "*.template.yaml"\n')
             f.flush()
@@ -513,7 +512,7 @@ class IgnoreConfigTestCase(unittest.TestCase):
         trailing = '[error] trailing spaces (trailing-spaces)'
         hyphen = '[error] too many spaces after hyphen (hyphens)'
 
-        self.assertEqual(out, '\n'.join((
+        self.assertEqual(out, rsep('\n'.join((
             './bin/file.lint-me-anyway.yaml:3:3: ' + keydup,
             './bin/file.lint-me-anyway.yaml:4:17: ' + trailing,
             './bin/file.lint-me-anyway.yaml:5:5: ' + hyphen,
@@ -547,10 +546,10 @@ class IgnoreConfigTestCase(unittest.TestCase):
             './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:3:3: ' + keydup,
             './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:4:17: ' + trailing,
             './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:5:5: ' + hyphen,
-        )))
+        ))))
 
     def test_run_with_ignore_str(self):
-        with open(os.path.join(self.wd, '.yamllint'), 'w') as f:
+        with open(os.path.join(self.wd, '.yamllint'), 'w', newline='') as f:
             f.write('extends: default\n'
                     'ignore: |\n'
                     '  *.dont-lint-me.yaml\n'
@@ -577,7 +576,7 @@ class IgnoreConfigTestCase(unittest.TestCase):
         trailing = '[error] trailing spaces (trailing-spaces)'
         hyphen = '[error] too many spaces after hyphen (hyphens)'
 
-        self.assertEqual(out, '\n'.join((
+        self.assertEqual(out, rsep('\n'.join((
             './.yamllint:1:1: ' + docstart,
             './bin/file.lint-me-anyway.yaml:3:3: ' + keydup,
             './bin/file.lint-me-anyway.yaml:4:17: ' + trailing,
@@ -601,10 +600,10 @@ class IgnoreConfigTestCase(unittest.TestCase):
             './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:3:3: ' + keydup,
             './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:4:17: ' + trailing,
             './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:5:5: ' + hyphen,
-        )))
+        ))))
 
     def test_run_with_ignore_list(self):
-        with open(os.path.join(self.wd, '.yamllint'), 'w') as f:
+        with open(os.path.join(self.wd, '.yamllint'), 'w', newline='') as f:
             f.write('extends: default\n'
                     'ignore:\n'
                     '  - "*.dont-lint-me.yaml"\n'
@@ -631,7 +630,7 @@ class IgnoreConfigTestCase(unittest.TestCase):
         trailing = '[error] trailing spaces (trailing-spaces)'
         hyphen = '[error] too many spaces after hyphen (hyphens)'
 
-        self.assertEqual(out, '\n'.join((
+        self.assertEqual(out, rsep('\n'.join((
             './.yamllint:1:1: ' + docstart,
             './bin/file.lint-me-anyway.yaml:3:3: ' + keydup,
             './bin/file.lint-me-anyway.yaml:4:17: ' + trailing,
@@ -655,13 +654,13 @@ class IgnoreConfigTestCase(unittest.TestCase):
             './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:3:3: ' + keydup,
             './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:4:17: ' + trailing,
             './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:5:5: ' + hyphen,
-        )))
+        ))))
 
     def test_run_with_ignore_from_file(self):
-        with open(os.path.join(self.wd, '.yamllint'), 'w') as f:
+        with open(os.path.join(self.wd, '.yamllint'), 'w', newline='') as f:
             f.write('extends: default\n'
                     'ignore-from-file: .gitignore\n')
-        with open(os.path.join(self.wd, '.gitignore'), 'w') as f:
+        with open(os.path.join(self.wd, '.gitignore'), 'w', newline='') as f:
             f.write('*.dont-lint-me.yaml\n'
                     '/bin/\n'
                     '!/bin/*.lint-me-anyway.yaml\n')
@@ -678,7 +677,7 @@ class IgnoreConfigTestCase(unittest.TestCase):
         trailing = '[error] trailing spaces (trailing-spaces)'
         hyphen = '[error] too many spaces after hyphen (hyphens)'
 
-        self.assertEqual(out, '\n'.join((
+        self.assertEqual(out, rsep('\n'.join((
             './.yamllint:1:1: ' + docstart,
             './bin/file.lint-me-anyway.yaml:3:3: ' + keydup,
             './bin/file.lint-me-anyway.yaml:4:17: ' + trailing,
@@ -707,16 +706,16 @@ class IgnoreConfigTestCase(unittest.TestCase):
             './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:3:3: ' + keydup,
             './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:4:17: ' + trailing,
             './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:5:5: ' + hyphen,
-        )))
+        ))))
 
     def test_run_with_ignored_from_file(self):
-        with open(os.path.join(self.wd, '.yamllint'), 'w') as f:
+        with open(os.path.join(self.wd, '.yamllint'), 'w', newline='') as f:
             f.write('ignore-from-file: [.gitignore, .yamlignore]\n'
                     'extends: default\n')
-        with open(os.path.join(self.wd, '.gitignore'), 'w') as f:
+        with open(os.path.join(self.wd, '.gitignore'), 'w', newline='') as f:
             f.write('*.dont-lint-me.yaml\n'
                     '/bin/\n')
-        with open(os.path.join(self.wd, '.yamlignore'), 'w') as f:
+        with open(os.path.join(self.wd, '.yamlignore'), 'w', newline='') as f:
             f.write('!/bin/*.lint-me-anyway.yaml\n')
 
         sys.stdout = StringIO()
@@ -731,7 +730,7 @@ class IgnoreConfigTestCase(unittest.TestCase):
         trailing = '[error] trailing spaces (trailing-spaces)'
         hyphen = '[error] too many spaces after hyphen (hyphens)'
 
-        self.assertEqual(out, '\n'.join((
+        self.assertEqual(out, rsep('\n'.join((
             './.yamllint:1:1: ' + docstart,
             './bin/file.lint-me-anyway.yaml:3:3: ' + keydup,
             './bin/file.lint-me-anyway.yaml:4:17: ' + trailing,
@@ -760,4 +759,4 @@ class IgnoreConfigTestCase(unittest.TestCase):
             './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:3:3: ' + keydup,
             './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:4:17: ' + trailing,
             './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:5:5: ' + hyphen,
-        )))
+        ))))

tests/test_module.py

@@ -20,6 +20,8 @@ import tempfile
 import sys
 import unittest
 
+from tests.common import rsep
+
 
 PYTHON = sys.executable or 'python'
 
@@ -29,12 +31,13 @@ class ModuleTestCase(unittest.TestCase):
         self.wd = tempfile.mkdtemp(prefix='yamllint-tests-')
 
         # file with only one warning
-        with open(os.path.join(self.wd, 'warn.yaml'), 'w') as f:
+        with open(os.path.join(self.wd, 'warn.yaml'), 'w', newline='') as f:
             f.write('key: value\n')
 
         # file in dir
         os.mkdir(os.path.join(self.wd, 'sub'))
-        with open(os.path.join(self.wd, 'sub', 'nok.yaml'), 'w') as f:
+        with open(os.path.join(self.wd, 'sub', 'nok.yaml'),
+                  'w', newline='') as f:
             f.write('---\n'
                     'list: [  1, 1, 2, 3, 5, 8]  \n')
 
@@ -44,41 +47,43 @@ class ModuleTestCase(unittest.TestCase):
     def test_run_module_no_args(self):
         with self.assertRaises(subprocess.CalledProcessError) as ctx:
             subprocess.check_output([PYTHON, '-m', 'yamllint'],
-                                    stderr=subprocess.STDOUT)
+                                    stderr=subprocess.STDOUT, text=True)
         self.assertEqual(ctx.exception.returncode, 2)
-        self.assertRegex(ctx.exception.output.decode(), r'^usage: yamllint')
+        self.assertRegex(ctx.exception.output, r'^usage: yamllint')
 
     def test_run_module_on_bad_dir(self):
         with self.assertRaises(subprocess.CalledProcessError) as ctx:
             subprocess.check_output([PYTHON, '-m', 'yamllint',
                                      '/does/not/exist'],
-                                    stderr=subprocess.STDOUT)
+                                    stderr=subprocess.STDOUT, text=True)
-        self.assertRegex(ctx.exception.output.decode(),
+        self.assertRegex(ctx.exception.output,
                          r'No such file or directory')
 
     def test_run_module_on_file(self):
         out = subprocess.check_output(
-            [PYTHON, '-m', 'yamllint', os.path.join(self.wd, 'warn.yaml')])
+            [PYTHON, '-m', 'yamllint', os.path.join(self.wd, 'warn.yaml')],
-        lines = out.decode().splitlines()
+            text=True)
-        self.assertIn('/warn.yaml', lines[0])
+        lines = out.splitlines()
+        self.assertIn(rsep('/warn.yaml'), lines[0])
         self.assertEqual('\n'.join(lines[1:]),
                          '  1:1       warning  missing document start "---"'
                          '  (document-start)\n')
 
     def test_run_module_on_dir(self):
         with self.assertRaises(subprocess.CalledProcessError) as ctx:
-            subprocess.check_output([PYTHON, '-m', 'yamllint', self.wd])
+            subprocess.check_output([PYTHON, '-m', 'yamllint', self.wd],
+                                    text=True)
         self.assertEqual(ctx.exception.returncode, 1)
 
-        files = ctx.exception.output.decode().split('\n\n')
+        files = ctx.exception.output.split('\n\n')
         self.assertIn(
-            '/warn.yaml\n'
+            rsep('/warn.yaml\n'
-            '  1:1       warning  missing document start "---"'
+                 '  1:1       warning  missing document start "---"'
-            '  (document-start)',
+                 '  (document-start)'),
             files[0])
         self.assertIn(
-            '/sub/nok.yaml\n'
+            rsep('/sub/nok.yaml\n'
-            '  2:9       error    too many spaces inside brackets'
+                 '  2:9       error    too many spaces inside brackets'
-            '  (brackets)\n'
+                 '  (brackets)\n'
-            '  2:27      error    trailing spaces  (trailing-spaces)',
+                 '  2:27      error    trailing spaces  (trailing-spaces)'),
             files[1])

tests/test_unicode.py

@@ -0,0 +1,192 @@
+# vim:set sw=4 ts=8 et fileencoding=utf8:
+# Copyright (C) 2023 Serguei E. Leontiev
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import io
+import locale
+import os
+import shutil
+import sys
+import unittest
+
+from tests.common import build_temp_workspace
+
+from yamllint import linter
+from yamllint.config import YamlLintConfig
+
+CONFIG = """
+extends: default
+"""
+
+GREEK = """---
+greek:
+  8: [Θ, θ, θήτα, [тета], Т]
+  20: [Υ, υ, ύψιλον, [ипсилон], И]
+"""
+GREEK_P = set([('document-end', 4)])
+
+CP1252 = """---
+capitals:
+  1: Reykjavík
+  2: Tórshavn
+"""
+CP1252_P = set([('unicode-decode', 0)])
+
+MINIMAL = "m:\n"
+MINIMAL_P = set([('document-start', 1),
+                 ('document-end', 1)])
+
+FIRST = """Θ:\n"""
+FIRST_P = set([('unicode-first-not-ascii', 1),
+               ('document-start', 1),
+               ('document-end', 1)])
+
+ENC = ['utf-8', 'utf-16le', 'utf-16be', 'utf-32le', 'utf-32be']
+
+
+class UnicodeTestCase(unittest.TestCase):
+    @classmethod
+    def fn(cls, enc: str, bom: bool) -> str:
+        return os.path.join(cls.wd, enc + ("-bom" if bom else "") + ".yml")
+
+    @classmethod
+    def create_file(cls, body: str, enc: str, bom: bool) -> None:
+        with open(cls.fn(enc, bom), 'w', encoding=enc) as f:
+            f.write(("\uFEFF" if bom else "") + body)
+
+    @classmethod
+    def setUpClass(cls):
+        super(UnicodeTestCase, cls).setUpClass()
+
+        cls.slc = locale.getlocale(locale.LC_ALL)
+        cls.cfg = YamlLintConfig('extends: default\n'
+                                 'rules: {document-end: {level: warning}}\n'
+                                 )
+        cls.wd = build_temp_workspace({})
+        for enc in ENC:
+            cls.create_file(GREEK, enc, True)
+            cls.create_file(GREEK, enc, False)
+        cls.create_file(GREEK, 'utf-7', True)
+        cls.create_file(CP1252, 'cp1252', False)
+        cls.create_file(MINIMAL, 'ascii', False)
+
+    @classmethod
+    def tearDownClass(cls):
+        super(UnicodeTestCase, cls).tearDownClass()
+
+        shutil.rmtree(cls.wd)
+        locale.setlocale(locale.LC_ALL, cls.slc)
+
+    def run_fobj(self, fobj, exp):
+        ep = exp.copy()
+        pcnt = 0
+        for p in linter.run(fobj, self.cfg):
+            if (p.rule, p.line) in ep:
+                ep.remove((p.rule, p.line),)
+            else:
+                print('UnicodeTestCase', p.desc, p.line, p.rule)
+                pcnt += 1
+        self.assertEqual(len(ep), 0)
+        self.assertEqual(pcnt, 0)
+
+    def run_file(self, lc, enc, bom, exp):
+        try:
+            locale.setlocale(locale.LC_ALL, lc)
+            with open(self.fn(enc, bom)) as f:
+                self.run_fobj(f, exp)
+            locale.setlocale(locale.LC_ALL, self.slc)
+        except locale.Error:
+            self.skipTest('locale ' + lc + ' not available')
+
+    def run_bytes(self, body, enc, bom, buf, exp):
+        bs = (("\uFEFF" if bom else "") + body).encode(enc)
+        if buf:
+            self.run_fobj(io.TextIOWrapper(io.BufferedReader(io.BytesIO(bs))),
+                          exp)
+        else:
+            self.run_fobj(io.TextIOWrapper(io.BytesIO(bs)), exp)
+
+    def test_file_en_US_UTF_8_utf8_nob(self):
+        self.run_file('en_US.UTF-8', 'utf-8', False, GREEK_P)
+
+    def test_file_ru_RU_CP1251_utf8_nob(self):
+        self.run_file('ru_RU.CP1251', 'utf-8', False, GREEK_P)
+
+    def test_file_en_US_utf8_cp1252(self):
+        self.run_file('en_US.utf8' if sys.platform.startswith('linux')
+                      else 'en_US.UTF-8',
+                      'cp1252', False, CP1252_P)
+
+    def test_file_en_US_ISO8859_1_cp1252(self):
+        self.run_file('en_US.ISO8859-1', 'cp1252', False, CP1252_P)
+
+    def test_file_C_utf8_nob(self):
+        self.run_file('C', 'utf-8', False, GREEK_P)
+
+    def test_file_C_utf8(self):
+        self.run_file('C', 'utf-8', True, GREEK_P)
+
+    def test_file_C_utf16le_nob(self):
+        self.run_file('C', 'utf-16le', False, GREEK_P)
+
+    def test_file_C_utf16le(self):
+        self.run_file('C', 'utf-16le', True, GREEK_P)
+
+    def test_file_C_utf16be_nob(self):
+        self.run_file('C', 'utf-16be', False, GREEK_P)
+
+    def test_file_C_utf16be(self):
+        self.run_file('C', 'utf-16be', True, GREEK_P)
+
+    def test_file_C_utf32le_nob(self):
+        self.run_file('C', 'utf-32le', False, GREEK_P)
+
+    def test_file_C_utf32le(self):
+        self.run_file('C', 'utf-32le', True, GREEK_P)
+
+    def test_file_C_utf32be_nob(self):
+        self.run_file('C', 'utf-32be', False, GREEK_P)
+
+    def test_file_C_utf32be(self):
+        self.run_file('C', 'utf-32be', True, GREEK_P)
+
+    def test_file_C_utf7(self):
+        self.run_file('C', 'utf-7', True, GREEK_P)
+
+    def test_file_minimal_nob(self):
+        self.run_file('C', 'ascii', False, MINIMAL_P)
+
+    def test_bytes_utf8_nob(self):
+        self.run_bytes(GREEK, 'utf-8', False, False, GREEK_P)
+
+    def test_bytes_utf16(self):
+        # .encode('utf-16') insert BOM automatically
+        self.run_bytes(GREEK, 'utf-16', False, False, GREEK_P)
+
+    def test_bytes_utf32_buf(self):
+        # .encode('utf-32') insert BOM automatically
+        self.run_bytes(GREEK, 'utf-32', False, True, GREEK_P)
+
+    def test_bytes_minimal_nob(self):
+        self.run_bytes(MINIMAL, 'ascii', False, False, MINIMAL_P)
+
+    def test_bytes_minimal_nob_buf(self):
+        self.run_bytes(MINIMAL, 'ascii', False, True, MINIMAL_P)
+
+    def test_bytes_first_nob(self):
+        self.run_bytes(FIRST, 'utf-8', False, False, FIRST_P)
+
+    def test_bytes_first_nob_buf(self):
+        self.run_bytes(FIRST, 'utf-8', False, True, FIRST_P)

yamllint/cli.py

@@ -232,11 +232,13 @@ def run(argv=None):
         try:
             with open(file, newline='') as f:
                 problems = linter.run(f, conf, filepath)
+                prob_level = show_problems(problems,
+                                           file,
+                                           args_format=args.format,
+                                           no_warn=args.no_warnings)
         except OSError as e:
             print(e, file=sys.stderr)
             sys.exit(-1)
-        prob_level = show_problems(problems, file, args_format=args.format,
-                                   no_warn=args.no_warnings)
         max_level = max(max_level, prob_level)
 
     # read yaml from stdin

yamllint/linter.py

@@ -13,14 +13,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import re
+import codecs
 import io
+import re
 
 import yaml
 
 from yamllint import parser
 
-
 PROBLEM_LEVELS = {
     0: None,
     1: 'warning',
@@ -185,14 +185,91 @@ def get_syntax_error(buffer):
         return problem
 
 
-def _run(buffer, conf, filepath):
+def _read_yaml_unicode(f: io.IOBase) -> str:
-    assert hasattr(buffer, '__getitem__'), \
+    """Reads and decodes file as p.5.2. Character Encodings
-        '_run() argument must be a buffer, not a stream'
+
+       Parameters
+       ----------
+       f:
+           For CLI - file open for reading in text mode
+           (TextIOWrapper(BufferedReader(FileIO)))
+
+           For API & tests - may be text or binary file object
+           (StringIO, TextIOWrapper(BytesIO) or
+           TextIOWrapper(BufferedReader(BytesIO)))
+    """
+    if not isinstance(f, io.TextIOWrapper):
+        # StringIO already have unicode, don't need decode
+        return (f.read(), False)
+
+    b = f.buffer
+    need = 4
+    if not isinstance(b, io.BufferedReader):
+        bs = bytes(b.getbuffer()[:need])  # BytesIO don't need peek()
+    else:
+        # Maximum of 4 raw.read()'s non-blocking file (or pipe)
+        # are required for peek 4 bytes or achieve EOF
+        lpbs = 0
+        bs = b.peek(need)
+        while len(bs) < need and len(bs) > lpbs:
+            # len(bs) > lpbs <=> b.raw.read() returned some bytes, not EOF
+            lpbs = len(bs)
+            bs = b.peek(need)
+        assert len(bs) >= need or not b.raw.read(1)
+
+    if bs.startswith(codecs.BOM_UTF32_BE):
+        f.reconfigure(encoding='utf-32be', errors='strict')
+    elif bs.startswith(codecs.BOM_UTF32_LE):
+        f.reconfigure(encoding='utf-32le', errors='strict')
+    elif bs.startswith(codecs.BOM_UTF16_BE):
+        f.reconfigure(encoding='utf-16be', errors='strict')
+    elif bs.startswith(codecs.BOM_UTF16_LE):
+        f.reconfigure(encoding='utf-16le', errors='strict')
+    elif bs.startswith(codecs.BOM_UTF8):
+        f.reconfigure(encoding='utf-8', errors='strict')
+    elif bs.startswith(b'+/v8'):
+        f.reconfigure(encoding='utf-7', errors='strict')
+    else:
+        if len(bs) >= 4 and bs[:3] == b'\x00\x00\x00' and bs[3]:
+            f.reconfigure(encoding='utf-32be', errors='strict')
+        elif len(bs) >= 4 and bs[0] and bs[1:4] == b'\x00\x00\x00':
+            f.reconfigure(encoding='utf-32le', errors='strict')
+        elif len(bs) >= 2 and bs[0] == 0 and bs[1]:
+            f.reconfigure(encoding='utf-16be', errors='strict')
+        elif len(bs) >= 2 and bs[0] and bs[1] == 0:
+            f.reconfigure(encoding='utf-16le', errors='strict')
+        else:
+            f.reconfigure(encoding='utf-8', errors='strict')
+        return (f.read(), False)
+    initial_bom = f.read(1)
+    assert initial_bom == '\uFEFF'
+    return (f.read(), True)
+
+
+def _run(input, conf, filepath):
+    if isinstance(input, str):
+        buffer, initial_bom = input, False
+    else:
+        try:
+            buffer, initial_bom = _read_yaml_unicode(input)
+        except UnicodeDecodeError as e:
+            problem = LintProblem(0, 0, str(e), 'unicode-decode')
+            problem.level = 'error'
+            yield problem
+            return
 
     first_line = next(parser.line_generator(buffer)).content
     if re.match(r'^#\s*yamllint disable-file\s*$', first_line):
         return
 
+    if not initial_bom and first_line and not (first_line[0].isascii() and
+       (first_line[0].isprintable() or first_line[0].isspace())):
+        problem = LintProblem(1, 1,
+                              "First Unicode character not ASCII without BOM",
+                              'unicode-first-not-ascii')
+        problem.level = 'warning'
+        yield problem
+
     # If the document contains a syntax error, save it and yield it at the
     # right line
     syntax_error = get_syntax_error(buffer)
@@ -226,11 +303,10 @@ def run(input, conf, filepath=None):
     if filepath is not None and conf.is_file_ignored(filepath):
         return ()
 
-    if isinstance(input, (bytes, str)):
+    if isinstance(input, str):
         return _run(input, conf, filepath)
-    elif isinstance(input, io.IOBase):
+    if isinstance(input, bytes):
-        # We need to have everything in memory to parse correctly
+        input = io.TextIOWrapper(io.BytesIO(input))
-        content = input.read()
+    if isinstance(input, io.IOBase):
-        return _run(content, conf, filepath)
+        return _run(input, conf, filepath)
-    else:
+    raise TypeError('input should be a string or a stream')
-        raise TypeError('input should be a string or a stream')