diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8c1b616..18fc51c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -33,16 +33,21 @@ jobs: test: name: Tests - runs-on: ubuntu-latest strategy: fail-fast: false matrix: + os: ['macos', 'ubuntu', 'windows'] python-version: - '3.7' - '3.8' - '3.9' - '3.10' - '3.11' + runs-on: ${{ matrix.os }}-latest + defaults: + run: + shell: bash -e {0} + steps: - name: Checkout uses: actions/checkout@v3 @@ -50,12 +55,20 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - run: pip install . - name: Append GitHub Actions system path + if: ${{ matrix.os == 'ubuntu' }} run: echo "$HOME/.local/bin" >> $GITHUB_PATH - run: pip install coverage - - run: pip install . + if: ${{ matrix.os == 'ubuntu' }} # https://github.com/AndreMiras/coveralls-python-action/issues/18 - run: echo -e "[run]\nrelative_files = True" > .coveragerc + if: ${{ matrix.os == 'ubuntu' }} - run: coverage run -m unittest discover + if: ${{ matrix.os == 'ubuntu' }} - name: Coveralls + if: ${{ matrix.os == 'ubuntu' }} uses: AndreMiras/coveralls-python-action@develop + - name: Unittests only + if: ${{ matrix.os != 'ubuntu' }} + run: python -m unittest diff --git a/tests/common.py b/tests/common.py index 65af63b..547ea02 100644 --- a/tests/common.py +++ b/tests/common.py @@ -65,9 +65,12 @@ def build_temp_workspace(files): if type(content) is list: os.mkdir(path) else: - mode = 'wb' if isinstance(content, bytes) else 'w' - with open(path, mode) as f: - f.write(content) + if isinstance(content, bytes): + with open(path, 'wb') as f: + f.write(content) + else: + with open(path, 'w', newline='') as f: + f.write(content) return tempdir @@ -84,3 +87,18 @@ def temp_workspace(files): finally: os.chdir(backup_wd) shutil.rmtree(wd) + + +@contextlib.contextmanager +def CompatNamedTemporaryFile(*args, **kwargs): + try: + assert 'delete' not in kwargs, "not applicable" + f = tempfile.NamedTemporaryFile(*args, **kwargs, delete=False) + yield f + finally: + f.close() + os.unlink(f.name) + + +def rsep(s: str) -> str: + return s.replace('/', os.sep) diff --git a/tests/rules/test_key_ordering.py b/tests/rules/test_key_ordering.py index 7d17603..5d05861 100644 --- a/tests/rules/test_key_ordering.py +++ b/tests/rules/test_key_ordering.py @@ -116,6 +116,8 @@ class KeyOrderingTestCase(RuleTestCase): self.addCleanup(locale.setlocale, locale.LC_ALL, (None, None)) try: locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') + if not locale.strcoll('t', 'T') < 0: # pragma: no cover + self.skipTest("Not 't' < 'T' for locale en_US.UTF-8") except locale.Error: # pragma: no cover self.skipTest('locale en_US.UTF-8 not available') conf = ('key-ordering: enable') diff --git a/tests/test_cli.py b/tests/test_cli.py index 419af92..7c260a2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -14,21 +14,25 @@ # along with this program. If not, see . from io import StringIO -import fcntl import locale import os -import pty import shutil import sys import tempfile import unittest -from tests.common import build_temp_workspace, temp_workspace +from tests.common import (build_temp_workspace, temp_workspace, + CompatNamedTemporaryFile, rsep) from yamllint import cli from yamllint import config +if not sys.platform.startswith('win'): + import pty + import fcntl + + class RunContext: """Context manager for ``cli.run()`` to capture exit code and streams.""" @@ -127,9 +131,10 @@ class CommandLineTestCase(unittest.TestCase): os.path.join(self.wd, 'dos.yml'), os.path.join(self.wd, 'empty.yml'), os.path.join(self.wd, 'en.yaml'), - os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'), - os.path.join(self.wd, 'sub/directory.yaml/empty.yml'), - os.path.join(self.wd, 'sub/ok.yaml'), + os.path.join(self.wd, + rsep('s/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml')), + os.path.join(self.wd, rsep('sub/directory.yaml/empty.yml')), + os.path.join(self.wd, rsep('sub/ok.yaml')), os.path.join(self.wd, 'warn.yaml')], ) @@ -145,7 +150,8 @@ class CommandLineTestCase(unittest.TestCase): self.assertEqual( sorted(cli.find_files_recursively(items, conf)), [os.path.join(self.wd, 'empty.yml'), - os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml')], + os.path.join(self.wd, + rsep('s/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'))], ) items = [os.path.join(self.wd, 'sub'), @@ -153,8 +159,8 @@ class CommandLineTestCase(unittest.TestCase): self.assertEqual( sorted(cli.find_files_recursively(items, conf)), [os.path.join(self.wd, '/etc/another/file'), - os.path.join(self.wd, 'sub/directory.yaml/empty.yml'), - os.path.join(self.wd, 'sub/ok.yaml')], + os.path.join(self.wd, rsep('sub/directory.yaml/empty.yml')), + os.path.join(self.wd, rsep('sub/ok.yaml'))], ) conf = config.YamlLintConfig('extends: default\n' @@ -165,8 +171,9 @@ class CommandLineTestCase(unittest.TestCase): [os.path.join(self.wd, 'a.yaml'), os.path.join(self.wd, 'c.yaml'), os.path.join(self.wd, 'en.yaml'), - os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'), - os.path.join(self.wd, 'sub/ok.yaml'), + os.path.join(self.wd, + rsep('s/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml')), + os.path.join(self.wd, rsep('sub/ok.yaml')), os.path.join(self.wd, 'warn.yaml')] ) @@ -177,7 +184,7 @@ class CommandLineTestCase(unittest.TestCase): sorted(cli.find_files_recursively([self.wd], conf)), [os.path.join(self.wd, 'dos.yml'), os.path.join(self.wd, 'empty.yml'), - os.path.join(self.wd, 'sub/directory.yaml/empty.yml')] + os.path.join(self.wd, rsep('sub/directory.yaml/empty.yml'))] ) conf = config.YamlLintConfig('extends: default\n' @@ -199,11 +206,12 @@ class CommandLineTestCase(unittest.TestCase): os.path.join(self.wd, 'empty.yml'), os.path.join(self.wd, 'en.yaml'), os.path.join(self.wd, 'no-yaml.json'), - os.path.join(self.wd, 'non-ascii/éçäγλνπ¥/utf-8'), - os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'), - os.path.join(self.wd, 'sub/directory.yaml/empty.yml'), - os.path.join(self.wd, 'sub/directory.yaml/not-yaml.txt'), - os.path.join(self.wd, 'sub/ok.yaml'), + os.path.join(self.wd, rsep('non-ascii/éçäγλνπ¥/utf-8')), + os.path.join(self.wd, + rsep('s/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml')), + os.path.join(self.wd, rsep('sub/directory.yaml/empty.yml')), + os.path.join(self.wd, rsep('sub/directory.yaml/not-yaml.txt')), + os.path.join(self.wd, rsep('sub/ok.yaml')), os.path.join(self.wd, 'warn.yaml')] ) @@ -220,11 +228,12 @@ class CommandLineTestCase(unittest.TestCase): os.path.join(self.wd, 'empty.yml'), os.path.join(self.wd, 'en.yaml'), os.path.join(self.wd, 'no-yaml.json'), - os.path.join(self.wd, 'non-ascii/éçäγλνπ¥/utf-8'), - os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'), - os.path.join(self.wd, 'sub/directory.yaml/empty.yml'), - os.path.join(self.wd, 'sub/directory.yaml/not-yaml.txt'), - os.path.join(self.wd, 'sub/ok.yaml'), + os.path.join(self.wd, rsep('non-ascii/éçäγλνπ¥/utf-8')), + os.path.join(self.wd, + rsep('s/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml')), + os.path.join(self.wd, rsep('sub/directory.yaml/empty.yml')), + os.path.join(self.wd, rsep('sub/directory.yaml/not-yaml.txt')), + os.path.join(self.wd, rsep('sub/ok.yaml')), os.path.join(self.wd, 'warn.yaml')] ) @@ -234,7 +243,7 @@ class CommandLineTestCase(unittest.TestCase): ' - \'**/utf-8\'\n') self.assertEqual( sorted(cli.find_files_recursively([self.wd], conf)), - [os.path.join(self.wd, 'non-ascii/éçäγλνπ¥/utf-8')] + [os.path.join(self.wd, rsep('non-ascii/éçäγλνπ¥/utf-8'))] ) def test_run_with_bad_arguments(self): @@ -306,6 +315,8 @@ class CommandLineTestCase(unittest.TestCase): cli.run(('-c', f.name, os.path.join(self.wd, 'a.yaml'))) self.assertEqual(ctx.returncode, 1) + @unittest.skipIf(sys.platform.startswith('win'), + 'TODO Windows override HOME unimplemented') @unittest.skipIf(os.environ.get('GITHUB_RUN_ID'), '$HOME not overridable') def test_run_with_user_global_config_file(self): home = os.path.join(self.wd, 'fake-home') @@ -346,7 +357,7 @@ class CommandLineTestCase(unittest.TestCase): def test_run_with_user_yamllint_config_file_in_env(self): self.addCleanup(os.environ.__delitem__, 'YAMLLINT_CONFIG_FILE') - with tempfile.NamedTemporaryFile('w') as f: + with CompatNamedTemporaryFile('w') as f: os.environ['YAMLLINT_CONFIG_FILE'] = f.name f.write('rules: {trailing-spaces: disable}') f.flush() @@ -354,7 +365,7 @@ class CommandLineTestCase(unittest.TestCase): cli.run((os.path.join(self.wd, 'a.yaml'), )) self.assertEqual(ctx.returncode, 0) - with tempfile.NamedTemporaryFile('w') as f: + with CompatNamedTemporaryFile('w') as f: os.environ['YAMLLINT_CONFIG_FILE'] = f.name f.write('rules: {trailing-spaces: enable}') f.flush() @@ -368,6 +379,8 @@ class CommandLineTestCase(unittest.TestCase): # as the first two runs don't use setlocale() try: locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') + if not locale.strcoll('a', 'A') < 0: # pragma: no cover + self.skipTest("Not 'a' < 'A' for locale en_US.UTF-8") except locale.Error: # pragma: no cover self.skipTest('locale en_US.UTF-8 not available') locale.setlocale(locale.LC_ALL, (None, None)) @@ -477,7 +490,7 @@ class CommandLineTestCase(unittest.TestCase): self.assertEqual((ctx.returncode, ctx.stderr), (1, '')) self.assertEqual(ctx.stdout, ( '%s:3:1: [error] duplication of key "key" in mapping ' - '(key-duplicates)\n') % path) + '(key-duplicates)\n') % rsep(path)) def test_run_piped_output_nocolor(self): path = os.path.join(self.wd, 'a.yaml') @@ -492,6 +505,8 @@ class CommandLineTestCase(unittest.TestCase): '(new-line-at-end-of-file)\n' '\n' % path)) + @unittest.skipIf(sys.platform.startswith('win'), + 'Windows pseudo-TTY unimplemented') def test_run_default_format_output_in_tty(self): path = os.path.join(self.wd, 'a.yaml') @@ -689,9 +704,10 @@ class CommandLineTestCase(unittest.TestCase): os.path.join(self.wd, 'dos.yml'), os.path.join(self.wd, 'empty.yml'), os.path.join(self.wd, 'en.yaml'), - os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'), - os.path.join(self.wd, 'sub/directory.yaml/empty.yml'), - os.path.join(self.wd, 'sub/ok.yaml'), + os.path.join(self.wd, + rsep('s/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml')), + os.path.join(self.wd, rsep('sub/directory.yaml/empty.yml')), + os.path.join(self.wd, rsep('sub/ok.yaml')), os.path.join(self.wd, 'warn.yaml')] ) @@ -705,9 +721,10 @@ class CommandLineTestCase(unittest.TestCase): os.path.join(self.wd, 'c.yaml'), os.path.join(self.wd, 'en.yaml'), os.path.join(self.wd, 'no-yaml.json'), - os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'), - os.path.join(self.wd, 'sub/directory.yaml/not-yaml.txt'), - os.path.join(self.wd, 'sub/ok.yaml'), + os.path.join(self.wd, + rsep('s/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml')), + os.path.join(self.wd, rsep('sub/directory.yaml/not-yaml.txt')), + os.path.join(self.wd, rsep('sub/ok.yaml')), os.path.join(self.wd, 'warn.yaml')] ) @@ -724,9 +741,10 @@ class CommandLineConfigTestCase(unittest.TestCase): with RunContext(self) as ctx: cli.run(('-f', 'parsable', '.')) - self.assertEqual((ctx.returncode, ctx.stdout, ctx.stderr), - (0, './a.yml:1:1: [warning] missing document ' - 'start "---" (document-start)\n', '')) + self.assertEqual( + (ctx.returncode, ctx.stdout, ctx.stderr), + (0, rsep('./a.yml:1:1: [warning] missing document ' + 'start "---" (document-start)\n'), '')) with temp_workspace({**workspace, **{conf_file: conf}}): with RunContext(self) as ctx: @@ -747,10 +765,11 @@ class CommandLineConfigTestCase(unittest.TestCase): os.chdir('a/b/c/d/e/f') cli.run(('-f', 'parsable', '.')) - self.assertEqual((ctx.returncode, ctx.stdout, ctx.stderr), - (0, './g/a.yml:1:1: [warning] missing ' - 'document start "---" (document-start)\n', - '')) + self.assertEqual( + (ctx.returncode, ctx.stdout, ctx.stderr), + (0, rsep('./g/a.yml:1:1: [warning] missing ' + 'document start "---" (document-start)\n'), + '')) with temp_workspace({**workspace, **{conf_file: conf}}): with RunContext(self) as ctx: @@ -783,15 +802,17 @@ class CommandLineConfigTestCase(unittest.TestCase): os.chdir('a/b/c') cli.run(('-f', 'parsable', '.')) - self.assertEqual((ctx.returncode, ctx.stdout, ctx.stderr), - (0, './3spaces.yml:2:4: [warning] wrong indentation: ' - 'expected 4 but found 3 (indentation)\n', '')) + self.assertEqual( + (ctx.returncode, ctx.stdout, ctx.stderr), + (0, rsep('./3spaces.yml:2:4: [warning] wrong indentation: ' + 'expected 4 but found 3 (indentation)\n'), '')) with temp_workspace({**workspace, **{'a/b/.yamllint.yml': conf3}}): with RunContext(self) as ctx: os.chdir('a/b/c') cli.run(('-f', 'parsable', '.')) - self.assertEqual((ctx.returncode, ctx.stdout, ctx.stderr), - (0, './4spaces.yml:2:5: [warning] wrong indentation: ' - 'expected 3 but found 4 (indentation)\n', '')) + self.assertEqual( + (ctx.returncode, ctx.stdout, ctx.stderr), + (0, rsep('./4spaces.yml:2:5: [warning] wrong indentation: ' + 'expected 3 but found 4 (indentation)\n'), '')) diff --git a/tests/test_config.py b/tests/test_config.py index 8e90246..a601f8a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -17,10 +17,9 @@ from io import StringIO import os import shutil import sys -import tempfile import unittest -from tests.common import build_temp_workspace +from tests.common import build_temp_workspace, CompatNamedTemporaryFile, rsep from yamllint.config import YamlLintConfigError from yamllint import cli @@ -245,7 +244,7 @@ class ExtendedConfigTestCase(unittest.TestCase): self.assertEqual(len(new.enabled_rules(None)), 2) def test_extend_on_file(self): - with tempfile.NamedTemporaryFile('w') as f: + with CompatNamedTemporaryFile('w') as f: f.write('rules:\n' ' colons:\n' ' max-spaces-before: 0\n' @@ -264,7 +263,7 @@ class ExtendedConfigTestCase(unittest.TestCase): self.assertEqual(len(c.enabled_rules(None)), 2) def test_extend_remove_rule(self): - with tempfile.NamedTemporaryFile('w') as f: + with CompatNamedTemporaryFile('w') as f: f.write('rules:\n' ' colons:\n' ' max-spaces-before: 0\n' @@ -283,7 +282,7 @@ class ExtendedConfigTestCase(unittest.TestCase): self.assertEqual(len(c.enabled_rules(None)), 1) def test_extend_edit_rule(self): - with tempfile.NamedTemporaryFile('w') as f: + with CompatNamedTemporaryFile('w') as f: f.write('rules:\n' ' colons:\n' ' max-spaces-before: 0\n' @@ -305,7 +304,7 @@ class ExtendedConfigTestCase(unittest.TestCase): self.assertEqual(len(c.enabled_rules(None)), 2) def test_extend_reenable_rule(self): - with tempfile.NamedTemporaryFile('w') as f: + with CompatNamedTemporaryFile('w') as f: f.write('rules:\n' ' colons:\n' ' max-spaces-before: 0\n' @@ -325,7 +324,7 @@ class ExtendedConfigTestCase(unittest.TestCase): self.assertEqual(len(c.enabled_rules(None)), 2) def test_extend_recursive_default_values(self): - with tempfile.NamedTemporaryFile('w') as f: + with CompatNamedTemporaryFile('w') as f: f.write('rules:\n' ' braces:\n' ' max-spaces-inside: 1248\n') @@ -340,7 +339,7 @@ class ExtendedConfigTestCase(unittest.TestCase): self.assertEqual(c.rules['braces']['min-spaces-inside-empty'], 2357) self.assertEqual(c.rules['braces']['max-spaces-inside-empty'], -1) - with tempfile.NamedTemporaryFile('w') as f: + with CompatNamedTemporaryFile('w') as f: f.write('rules:\n' ' colons:\n' ' max-spaces-before: 1337\n') @@ -352,8 +351,8 @@ class ExtendedConfigTestCase(unittest.TestCase): self.assertEqual(c.rules['colons']['max-spaces-before'], 1337) self.assertEqual(c.rules['colons']['max-spaces-after'], 1) - with tempfile.NamedTemporaryFile('w') as f1, \ - tempfile.NamedTemporaryFile('w') as f2: + with CompatNamedTemporaryFile('w') as f1, \ + CompatNamedTemporaryFile('w') as f2: f1.write('rules:\n' ' colons:\n' ' max-spaces-before: 1337\n') @@ -370,7 +369,7 @@ class ExtendedConfigTestCase(unittest.TestCase): self.assertEqual(c.rules['colons']['max-spaces-after'], 1) def test_extended_ignore_str(self): - with tempfile.NamedTemporaryFile('w') as f: + with CompatNamedTemporaryFile('w') as f: f.write('ignore: |\n' ' *.template.yaml\n') f.flush() @@ -380,7 +379,7 @@ class ExtendedConfigTestCase(unittest.TestCase): self.assertEqual(c.ignore.match_file('test.yaml'), False) def test_extended_ignore_list(self): - with tempfile.NamedTemporaryFile('w') as f: + with CompatNamedTemporaryFile('w') as f: f.write('ignore:\n' ' - "*.template.yaml"\n') f.flush() @@ -513,7 +512,7 @@ class IgnoreConfigTestCase(unittest.TestCase): trailing = '[error] trailing spaces (trailing-spaces)' hyphen = '[error] too many spaces after hyphen (hyphens)' - self.assertEqual(out, '\n'.join(( + self.assertEqual(out, rsep('\n'.join(( './bin/file.lint-me-anyway.yaml:3:3: ' + keydup, './bin/file.lint-me-anyway.yaml:4:17: ' + trailing, './bin/file.lint-me-anyway.yaml:5:5: ' + hyphen, @@ -547,10 +546,10 @@ class IgnoreConfigTestCase(unittest.TestCase): './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:3:3: ' + keydup, './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:4:17: ' + trailing, './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:5:5: ' + hyphen, - ))) + )))) def test_run_with_ignore_str(self): - with open(os.path.join(self.wd, '.yamllint'), 'w') as f: + with open(os.path.join(self.wd, '.yamllint'), 'w', newline='') as f: f.write('extends: default\n' 'ignore: |\n' ' *.dont-lint-me.yaml\n' @@ -577,7 +576,7 @@ class IgnoreConfigTestCase(unittest.TestCase): trailing = '[error] trailing spaces (trailing-spaces)' hyphen = '[error] too many spaces after hyphen (hyphens)' - self.assertEqual(out, '\n'.join(( + self.assertEqual(out, rsep('\n'.join(( './.yamllint:1:1: ' + docstart, './bin/file.lint-me-anyway.yaml:3:3: ' + keydup, './bin/file.lint-me-anyway.yaml:4:17: ' + trailing, @@ -601,10 +600,10 @@ class IgnoreConfigTestCase(unittest.TestCase): './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:3:3: ' + keydup, './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:4:17: ' + trailing, './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:5:5: ' + hyphen, - ))) + )))) def test_run_with_ignore_list(self): - with open(os.path.join(self.wd, '.yamllint'), 'w') as f: + with open(os.path.join(self.wd, '.yamllint'), 'w', newline='') as f: f.write('extends: default\n' 'ignore:\n' ' - "*.dont-lint-me.yaml"\n' @@ -631,7 +630,7 @@ class IgnoreConfigTestCase(unittest.TestCase): trailing = '[error] trailing spaces (trailing-spaces)' hyphen = '[error] too many spaces after hyphen (hyphens)' - self.assertEqual(out, '\n'.join(( + self.assertEqual(out, rsep('\n'.join(( './.yamllint:1:1: ' + docstart, './bin/file.lint-me-anyway.yaml:3:3: ' + keydup, './bin/file.lint-me-anyway.yaml:4:17: ' + trailing, @@ -655,13 +654,13 @@ class IgnoreConfigTestCase(unittest.TestCase): './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:3:3: ' + keydup, './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:4:17: ' + trailing, './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:5:5: ' + hyphen, - ))) + )))) def test_run_with_ignore_from_file(self): - with open(os.path.join(self.wd, '.yamllint'), 'w') as f: + with open(os.path.join(self.wd, '.yamllint'), 'w', newline='') as f: f.write('extends: default\n' 'ignore-from-file: .gitignore\n') - with open(os.path.join(self.wd, '.gitignore'), 'w') as f: + with open(os.path.join(self.wd, '.gitignore'), 'w', newline='') as f: f.write('*.dont-lint-me.yaml\n' '/bin/\n' '!/bin/*.lint-me-anyway.yaml\n') @@ -678,7 +677,7 @@ class IgnoreConfigTestCase(unittest.TestCase): trailing = '[error] trailing spaces (trailing-spaces)' hyphen = '[error] too many spaces after hyphen (hyphens)' - self.assertEqual(out, '\n'.join(( + self.assertEqual(out, rsep('\n'.join(( './.yamllint:1:1: ' + docstart, './bin/file.lint-me-anyway.yaml:3:3: ' + keydup, './bin/file.lint-me-anyway.yaml:4:17: ' + trailing, @@ -707,16 +706,16 @@ class IgnoreConfigTestCase(unittest.TestCase): './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:3:3: ' + keydup, './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:4:17: ' + trailing, './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:5:5: ' + hyphen, - ))) + )))) def test_run_with_ignored_from_file(self): - with open(os.path.join(self.wd, '.yamllint'), 'w') as f: + with open(os.path.join(self.wd, '.yamllint'), 'w', newline='') as f: f.write('ignore-from-file: [.gitignore, .yamlignore]\n' 'extends: default\n') - with open(os.path.join(self.wd, '.gitignore'), 'w') as f: + with open(os.path.join(self.wd, '.gitignore'), 'w', newline='') as f: f.write('*.dont-lint-me.yaml\n' '/bin/\n') - with open(os.path.join(self.wd, '.yamlignore'), 'w') as f: + with open(os.path.join(self.wd, '.yamlignore'), 'w', newline='') as f: f.write('!/bin/*.lint-me-anyway.yaml\n') sys.stdout = StringIO() @@ -731,7 +730,7 @@ class IgnoreConfigTestCase(unittest.TestCase): trailing = '[error] trailing spaces (trailing-spaces)' hyphen = '[error] too many spaces after hyphen (hyphens)' - self.assertEqual(out, '\n'.join(( + self.assertEqual(out, rsep('\n'.join(( './.yamllint:1:1: ' + docstart, './bin/file.lint-me-anyway.yaml:3:3: ' + keydup, './bin/file.lint-me-anyway.yaml:4:17: ' + trailing, @@ -760,4 +759,4 @@ class IgnoreConfigTestCase(unittest.TestCase): './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:3:3: ' + keydup, './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:4:17: ' + trailing, './s/s/ign-trail/s/s/file2.lint-me-anyway.yaml:5:5: ' + hyphen, - ))) + )))) diff --git a/tests/test_module.py b/tests/test_module.py index 299e153..1100f19 100644 --- a/tests/test_module.py +++ b/tests/test_module.py @@ -20,6 +20,8 @@ import tempfile import sys import unittest +from tests.common import rsep + PYTHON = sys.executable or 'python' @@ -29,12 +31,13 @@ class ModuleTestCase(unittest.TestCase): self.wd = tempfile.mkdtemp(prefix='yamllint-tests-') # file with only one warning - with open(os.path.join(self.wd, 'warn.yaml'), 'w') as f: + with open(os.path.join(self.wd, 'warn.yaml'), 'w', newline='') as f: f.write('key: value\n') # file in dir os.mkdir(os.path.join(self.wd, 'sub')) - with open(os.path.join(self.wd, 'sub', 'nok.yaml'), 'w') as f: + with open(os.path.join(self.wd, 'sub', 'nok.yaml'), + 'w', newline='') as f: f.write('---\n' 'list: [ 1, 1, 2, 3, 5, 8] \n') @@ -44,41 +47,43 @@ class ModuleTestCase(unittest.TestCase): def test_run_module_no_args(self): with self.assertRaises(subprocess.CalledProcessError) as ctx: subprocess.check_output([PYTHON, '-m', 'yamllint'], - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, text=True) self.assertEqual(ctx.exception.returncode, 2) - self.assertRegex(ctx.exception.output.decode(), r'^usage: yamllint') + self.assertRegex(ctx.exception.output, r'^usage: yamllint') def test_run_module_on_bad_dir(self): with self.assertRaises(subprocess.CalledProcessError) as ctx: subprocess.check_output([PYTHON, '-m', 'yamllint', '/does/not/exist'], - stderr=subprocess.STDOUT) - self.assertRegex(ctx.exception.output.decode(), + stderr=subprocess.STDOUT, text=True) + self.assertRegex(ctx.exception.output, r'No such file or directory') def test_run_module_on_file(self): out = subprocess.check_output( - [PYTHON, '-m', 'yamllint', os.path.join(self.wd, 'warn.yaml')]) - lines = out.decode().splitlines() - self.assertIn('/warn.yaml', lines[0]) + [PYTHON, '-m', 'yamllint', os.path.join(self.wd, 'warn.yaml')], + text=True) + lines = out.splitlines() + self.assertIn(rsep('/warn.yaml'), lines[0]) self.assertEqual('\n'.join(lines[1:]), ' 1:1 warning missing document start "---"' ' (document-start)\n') def test_run_module_on_dir(self): with self.assertRaises(subprocess.CalledProcessError) as ctx: - subprocess.check_output([PYTHON, '-m', 'yamllint', self.wd]) + subprocess.check_output([PYTHON, '-m', 'yamllint', self.wd], + text=True) self.assertEqual(ctx.exception.returncode, 1) - files = ctx.exception.output.decode().split('\n\n') + files = ctx.exception.output.split('\n\n') self.assertIn( - '/warn.yaml\n' - ' 1:1 warning missing document start "---"' - ' (document-start)', + rsep('/warn.yaml\n' + ' 1:1 warning missing document start "---"' + ' (document-start)'), files[0]) self.assertIn( - '/sub/nok.yaml\n' - ' 2:9 error too many spaces inside brackets' - ' (brackets)\n' - ' 2:27 error trailing spaces (trailing-spaces)', + rsep('/sub/nok.yaml\n' + ' 2:9 error too many spaces inside brackets' + ' (brackets)\n' + ' 2:27 error trailing spaces (trailing-spaces)'), files[1]) diff --git a/tests/test_unicode.py b/tests/test_unicode.py new file mode 100644 index 0000000..fe633d7 --- /dev/null +++ b/tests/test_unicode.py @@ -0,0 +1,192 @@ +# vim:set sw=4 ts=8 et fileencoding=utf8: +# Copyright (C) 2023 Serguei E. Leontiev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import io +import locale +import os +import shutil +import sys +import unittest + +from tests.common import build_temp_workspace + +from yamllint import linter +from yamllint.config import YamlLintConfig + +CONFIG = """ +extends: default +""" + +GREEK = """--- +greek: + 8: [Θ, θ, θήτα, [тета], Т] + 20: [Υ, υ, ύψιλον, [ипсилон], И] +""" +GREEK_P = set([('document-end', 4)]) + +CP1252 = """--- +capitals: + 1: Reykjavík + 2: Tórshavn +""" +CP1252_P = set([('unicode-decode', 0)]) + +MINIMAL = "m:\n" +MINIMAL_P = set([('document-start', 1), + ('document-end', 1)]) + +FIRST = """Θ:\n""" +FIRST_P = set([('unicode-first-not-ascii', 1), + ('document-start', 1), + ('document-end', 1)]) + +ENC = ['utf-8', 'utf-16le', 'utf-16be', 'utf-32le', 'utf-32be'] + + +class UnicodeTestCase(unittest.TestCase): + @classmethod + def fn(cls, enc: str, bom: bool) -> str: + return os.path.join(cls.wd, enc + ("-bom" if bom else "") + ".yml") + + @classmethod + def create_file(cls, body: str, enc: str, bom: bool) -> None: + with open(cls.fn(enc, bom), 'w', encoding=enc) as f: + f.write(("\uFEFF" if bom else "") + body) + + @classmethod + def setUpClass(cls): + super(UnicodeTestCase, cls).setUpClass() + + cls.slc = locale.getlocale(locale.LC_ALL) + cls.cfg = YamlLintConfig('extends: default\n' + 'rules: {document-end: {level: warning}}\n' + ) + cls.wd = build_temp_workspace({}) + for enc in ENC: + cls.create_file(GREEK, enc, True) + cls.create_file(GREEK, enc, False) + cls.create_file(GREEK, 'utf-7', True) + cls.create_file(CP1252, 'cp1252', False) + cls.create_file(MINIMAL, 'ascii', False) + + @classmethod + def tearDownClass(cls): + super(UnicodeTestCase, cls).tearDownClass() + + shutil.rmtree(cls.wd) + locale.setlocale(locale.LC_ALL, cls.slc) + + def run_fobj(self, fobj, exp): + ep = exp.copy() + pcnt = 0 + for p in linter.run(fobj, self.cfg): + if (p.rule, p.line) in ep: + ep.remove((p.rule, p.line),) + else: + print('UnicodeTestCase', p.desc, p.line, p.rule) + pcnt += 1 + self.assertEqual(len(ep), 0) + self.assertEqual(pcnt, 0) + + def run_file(self, lc, enc, bom, exp): + try: + locale.setlocale(locale.LC_ALL, lc) + with open(self.fn(enc, bom)) as f: + self.run_fobj(f, exp) + locale.setlocale(locale.LC_ALL, self.slc) + except locale.Error: + self.skipTest('locale ' + lc + ' not available') + + def run_bytes(self, body, enc, bom, buf, exp): + bs = (("\uFEFF" if bom else "") + body).encode(enc) + if buf: + self.run_fobj(io.TextIOWrapper(io.BufferedReader(io.BytesIO(bs))), + exp) + else: + self.run_fobj(io.TextIOWrapper(io.BytesIO(bs)), exp) + + def test_file_en_US_UTF_8_utf8_nob(self): + self.run_file('en_US.UTF-8', 'utf-8', False, GREEK_P) + + def test_file_ru_RU_CP1251_utf8_nob(self): + self.run_file('ru_RU.CP1251', 'utf-8', False, GREEK_P) + + def test_file_en_US_utf8_cp1252(self): + self.run_file('en_US.utf8' if sys.platform.startswith('linux') + else 'en_US.UTF-8', + 'cp1252', False, CP1252_P) + + def test_file_en_US_ISO8859_1_cp1252(self): + self.run_file('en_US.ISO8859-1', 'cp1252', False, CP1252_P) + + def test_file_C_utf8_nob(self): + self.run_file('C', 'utf-8', False, GREEK_P) + + def test_file_C_utf8(self): + self.run_file('C', 'utf-8', True, GREEK_P) + + def test_file_C_utf16le_nob(self): + self.run_file('C', 'utf-16le', False, GREEK_P) + + def test_file_C_utf16le(self): + self.run_file('C', 'utf-16le', True, GREEK_P) + + def test_file_C_utf16be_nob(self): + self.run_file('C', 'utf-16be', False, GREEK_P) + + def test_file_C_utf16be(self): + self.run_file('C', 'utf-16be', True, GREEK_P) + + def test_file_C_utf32le_nob(self): + self.run_file('C', 'utf-32le', False, GREEK_P) + + def test_file_C_utf32le(self): + self.run_file('C', 'utf-32le', True, GREEK_P) + + def test_file_C_utf32be_nob(self): + self.run_file('C', 'utf-32be', False, GREEK_P) + + def test_file_C_utf32be(self): + self.run_file('C', 'utf-32be', True, GREEK_P) + + def test_file_C_utf7(self): + self.run_file('C', 'utf-7', True, GREEK_P) + + def test_file_minimal_nob(self): + self.run_file('C', 'ascii', False, MINIMAL_P) + + def test_bytes_utf8_nob(self): + self.run_bytes(GREEK, 'utf-8', False, False, GREEK_P) + + def test_bytes_utf16(self): + # .encode('utf-16') insert BOM automatically + self.run_bytes(GREEK, 'utf-16', False, False, GREEK_P) + + def test_bytes_utf32_buf(self): + # .encode('utf-32') insert BOM automatically + self.run_bytes(GREEK, 'utf-32', False, True, GREEK_P) + + def test_bytes_minimal_nob(self): + self.run_bytes(MINIMAL, 'ascii', False, False, MINIMAL_P) + + def test_bytes_minimal_nob_buf(self): + self.run_bytes(MINIMAL, 'ascii', False, True, MINIMAL_P) + + def test_bytes_first_nob(self): + self.run_bytes(FIRST, 'utf-8', False, False, FIRST_P) + + def test_bytes_first_nob_buf(self): + self.run_bytes(FIRST, 'utf-8', False, True, FIRST_P) diff --git a/yamllint/cli.py b/yamllint/cli.py index d7fa156..a817b35 100644 --- a/yamllint/cli.py +++ b/yamllint/cli.py @@ -232,11 +232,13 @@ def run(argv=None): try: with open(file, newline='') as f: problems = linter.run(f, conf, filepath) + prob_level = show_problems(problems, + file, + args_format=args.format, + no_warn=args.no_warnings) except OSError as e: print(e, file=sys.stderr) sys.exit(-1) - prob_level = show_problems(problems, file, args_format=args.format, - no_warn=args.no_warnings) max_level = max(max_level, prob_level) # read yaml from stdin diff --git a/yamllint/linter.py b/yamllint/linter.py index 5501bb5..c919900 100644 --- a/yamllint/linter.py +++ b/yamllint/linter.py @@ -13,14 +13,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import re +import codecs import io +import re import yaml from yamllint import parser - PROBLEM_LEVELS = { 0: None, 1: 'warning', @@ -185,14 +185,91 @@ def get_syntax_error(buffer): return problem -def _run(buffer, conf, filepath): - assert hasattr(buffer, '__getitem__'), \ - '_run() argument must be a buffer, not a stream' +def _read_yaml_unicode(f: io.IOBase) -> str: + """Reads and decodes file as p.5.2. Character Encodings + + Parameters + ---------- + f: + For CLI - file open for reading in text mode + (TextIOWrapper(BufferedReader(FileIO))) + + For API & tests - may be text or binary file object + (StringIO, TextIOWrapper(BytesIO) or + TextIOWrapper(BufferedReader(BytesIO))) + """ + if not isinstance(f, io.TextIOWrapper): + # StringIO already have unicode, don't need decode + return (f.read(), False) + + b = f.buffer + need = 4 + if not isinstance(b, io.BufferedReader): + bs = bytes(b.getbuffer()[:need]) # BytesIO don't need peek() + else: + # Maximum of 4 raw.read()'s non-blocking file (or pipe) + # are required for peek 4 bytes or achieve EOF + lpbs = 0 + bs = b.peek(need) + while len(bs) < need and len(bs) > lpbs: + # len(bs) > lpbs <=> b.raw.read() returned some bytes, not EOF + lpbs = len(bs) + bs = b.peek(need) + assert len(bs) >= need or not b.raw.read(1) + + if bs.startswith(codecs.BOM_UTF32_BE): + f.reconfigure(encoding='utf-32be', errors='strict') + elif bs.startswith(codecs.BOM_UTF32_LE): + f.reconfigure(encoding='utf-32le', errors='strict') + elif bs.startswith(codecs.BOM_UTF16_BE): + f.reconfigure(encoding='utf-16be', errors='strict') + elif bs.startswith(codecs.BOM_UTF16_LE): + f.reconfigure(encoding='utf-16le', errors='strict') + elif bs.startswith(codecs.BOM_UTF8): + f.reconfigure(encoding='utf-8', errors='strict') + elif bs.startswith(b'+/v8'): + f.reconfigure(encoding='utf-7', errors='strict') + else: + if len(bs) >= 4 and bs[:3] == b'\x00\x00\x00' and bs[3]: + f.reconfigure(encoding='utf-32be', errors='strict') + elif len(bs) >= 4 and bs[0] and bs[1:4] == b'\x00\x00\x00': + f.reconfigure(encoding='utf-32le', errors='strict') + elif len(bs) >= 2 and bs[0] == 0 and bs[1]: + f.reconfigure(encoding='utf-16be', errors='strict') + elif len(bs) >= 2 and bs[0] and bs[1] == 0: + f.reconfigure(encoding='utf-16le', errors='strict') + else: + f.reconfigure(encoding='utf-8', errors='strict') + return (f.read(), False) + initial_bom = f.read(1) + assert initial_bom == '\uFEFF' + return (f.read(), True) + + +def _run(input, conf, filepath): + if isinstance(input, str): + buffer, initial_bom = input, False + else: + try: + buffer, initial_bom = _read_yaml_unicode(input) + except UnicodeDecodeError as e: + problem = LintProblem(0, 0, str(e), 'unicode-decode') + problem.level = 'error' + yield problem + return first_line = next(parser.line_generator(buffer)).content if re.match(r'^#\s*yamllint disable-file\s*$', first_line): return + if not initial_bom and first_line and not (first_line[0].isascii() and + (first_line[0].isprintable() or first_line[0].isspace())): + problem = LintProblem(1, 1, + "First Unicode character not ASCII without BOM", + 'unicode-first-not-ascii') + problem.level = 'warning' + yield problem + # If the document contains a syntax error, save it and yield it at the # right line syntax_error = get_syntax_error(buffer) @@ -226,11 +303,10 @@ def run(input, conf, filepath=None): if filepath is not None and conf.is_file_ignored(filepath): return () - if isinstance(input, (bytes, str)): + if isinstance(input, str): return _run(input, conf, filepath) - elif isinstance(input, io.IOBase): - # We need to have everything in memory to parse correctly - content = input.read() - return _run(content, conf, filepath) - else: - raise TypeError('input should be a string or a stream') + if isinstance(input, bytes): + input = io.TextIOWrapper(io.BytesIO(input)) + if isinstance(input, io.IOBase): + return _run(input, conf, filepath) + raise TypeError('input should be a string or a stream')