From 1b831f214a111dfb45a571fc40f4404bb6b5b62c Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Fri, 22 Dec 2023 17:46:06 -0600 Subject: [PATCH 01/68] Add new changelog template (#4125) --- CHANGES.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index d0c9e567457..526cbd12123 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,52 @@ # Change Log +## Unreleased + +### Highlights + + + +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + + ## 23.12.1 ### Packaging From 51786141cc4eb3c212be76638e66b91648d0e5f8 Mon Sep 17 00:00:00 2001 From: Anupya Pamidimukkala Date: Thu, 28 Dec 2023 01:23:42 -0500 Subject: [PATCH 02/68] Fix nits, chain comparisons, unused params, hyphens (#4114) --- src/black/brackets.py | 4 ++-- src/black/cache.py | 4 ++-- src/black/comments.py | 3 +-- src/black/linegen.py | 2 +- src/black/lines.py | 4 ++-- src/black/ranges.py | 5 +---- src/black/trans.py | 21 ++++++++++----------- 7 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/black/brackets.py b/src/black/brackets.py index 3020cc0d390..37e6b2590eb 100644 --- a/src/black/brackets.py +++ b/src/black/brackets.py @@ -115,7 +115,7 @@ def mark(self, leaf: Leaf) -> None: if delim and self.previous is not None: self.delimiters[id(self.previous)] = delim else: - delim = is_split_after_delimiter(leaf, self.previous) + delim = is_split_after_delimiter(leaf) if delim: self.delimiters[id(leaf)] = delim if leaf.type in OPENING_BRACKETS: @@ -215,7 +215,7 @@ def get_open_lsqb(self) -> Optional[Leaf]: return self.bracket_match.get((self.depth - 1, token.RSQB)) -def is_split_after_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> Priority: +def is_split_after_delimiter(leaf: Leaf) -> Priority: """Return the priority of the `leaf` delimiter, given a line break after it. The delimiter priorities returned here are from those delimiters that would diff --git a/src/black/cache.py b/src/black/cache.py index 6baa096baca..c844c37b6f8 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -58,9 +58,9 @@ class Cache: @classmethod def read(cls, mode: Mode) -> Self: - """Read the cache if it exists and is well formed. + """Read the cache if it exists and is well-formed. - If it is not well formed, the call to write later should + If it is not well-formed, the call to write later should resolve the issue. """ cache_file = get_cache_file(mode) diff --git a/src/black/comments.py b/src/black/comments.py index 25413121199..52bb024a799 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -221,8 +221,7 @@ def convert_one_fmt_off_pair( if comment.value in FMT_OFF: fmt_off_prefix = "" if len(lines) > 0 and not any( - comment_lineno >= line[0] and comment_lineno <= line[1] - for line in lines + line[0] <= comment_lineno <= line[1] for line in lines ): # keeping indentation of comment by preserving original whitespaces. fmt_off_prefix = prefix.split(comment.value)[0] diff --git a/src/black/linegen.py b/src/black/linegen.py index 245be235231..0fd4a8d9c96 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -1635,7 +1635,7 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf opening_bracket: Optional[Leaf] = None closing_bracket: Optional[Leaf] = None inner_brackets: Set[LeafID] = set() - for index, leaf, leaf_length in line.enumerate_with_length(reversed=True): + for index, leaf, leaf_length in line.enumerate_with_length(is_reversed=True): length += leaf_length if length > line_length: break diff --git a/src/black/lines.py b/src/black/lines.py index 2a41db173d4..0cd4189a778 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -451,7 +451,7 @@ def is_complex_subscript(self, leaf: Leaf) -> bool: ) def enumerate_with_length( - self, reversed: bool = False + self, is_reversed: bool = False ) -> Iterator[Tuple[Index, Leaf, int]]: """Return an enumeration of leaves with their length. @@ -459,7 +459,7 @@ def enumerate_with_length( """ op = cast( Callable[[Sequence[Leaf]], Iterator[Tuple[Index, Leaf]]], - enumerate_reversed if reversed else enumerate, + enumerate_reversed if is_reversed else enumerate, ) for index, leaf in op(self.leaves): length = len(leaf.prefix) + len(leaf.value) diff --git a/src/black/ranges.py b/src/black/ranges.py index 59e19242d47..06fa8790554 100644 --- a/src/black/ranges.py +++ b/src/black/ranges.py @@ -487,10 +487,7 @@ def _find_lines_mapping_index( index = start_index while index < len(lines_mappings): mapping = lines_mappings[index] - if ( - mapping.original_start <= original_line - and original_line <= mapping.original_end - ): + if mapping.original_start <= original_line <= mapping.original_end: return index index += 1 return index diff --git a/src/black/trans.py b/src/black/trans.py index ab3197fa6df..7c7335a005b 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -1273,7 +1273,7 @@ def iter_fexpr_spans(s: str) -> Iterator[Tuple[int, int]]: i += 1 continue - # if we're in an expression part of the f-string, fast forward through strings + # if we're in an expression part of the f-string, fast-forward through strings # note that backslashes are not legal in the expression portion of f-strings if stack: delim = None @@ -1740,7 +1740,7 @@ def passes_all_checks(i: Index) -> bool: """ Returns: True iff ALL of the conditions listed in the 'Transformations' - section of this classes' docstring would be be met by returning @i. + section of this classes' docstring would be met by returning @i. """ is_space = string[i] == " " is_split_safe = is_valid_index(i - 1) and string[i - 1] in SPLIT_SAFE_CHARS @@ -1932,7 +1932,7 @@ def _return_match(LL: List[Leaf]) -> Optional[int]: OR None, otherwise. """ - # If this line is apart of a return/yield statement and the first leaf + # If this line is a part of a return/yield statement and the first leaf # contains either the "return" or "yield" keywords... if parent_type(LL[0]) in [syms.return_stmt, syms.yield_expr] and LL[ 0 @@ -1957,7 +1957,7 @@ def _else_match(LL: List[Leaf]) -> Optional[int]: OR None, otherwise. """ - # If this line is apart of a ternary expression and the first leaf + # If this line is a part of a ternary expression and the first leaf # contains the "else" keyword... if ( parent_type(LL[0]) == syms.test @@ -1984,7 +1984,7 @@ def _assert_match(LL: List[Leaf]) -> Optional[int]: OR None, otherwise. """ - # If this line is apart of an assert statement and the first leaf + # If this line is a part of an assert statement and the first leaf # contains the "assert" keyword... if parent_type(LL[0]) == syms.assert_stmt and LL[0].value == "assert": is_valid_index = is_valid_index_factory(LL) @@ -2019,7 +2019,7 @@ def _assign_match(LL: List[Leaf]) -> Optional[int]: OR None, otherwise. """ - # If this line is apart of an expression statement or is a function + # If this line is a part of an expression statement or is a function # argument AND the first leaf contains a variable name... if ( parent_type(LL[0]) in [syms.expr_stmt, syms.argument, syms.power] @@ -2040,7 +2040,7 @@ def _assign_match(LL: List[Leaf]) -> Optional[int]: string_parser = StringParser() idx = string_parser.parse(LL, string_idx) - # The next leaf MAY be a comma iff this line is apart + # The next leaf MAY be a comma iff this line is a part # of a function argument... if ( parent_type(LL[0]) == syms.argument @@ -2187,8 +2187,7 @@ def do_transform( if opening_bracket is not None and opening_bracket in left_leaves: index = left_leaves.index(opening_bracket) if ( - index > 0 - and index < len(left_leaves) - 1 + 0 < index < len(left_leaves) - 1 and left_leaves[index - 1].type == token.COLON and left_leaves[index + 1].value == "lambda" ): @@ -2297,7 +2296,7 @@ def parse(self, leaves: List[Leaf], string_idx: int) -> int: * @leaves[@string_idx].type == token.STRING Returns: - The index directly after the last leaf which is apart of the string + The index directly after the last leaf which is a part of the string trailer, if a "trailer" exists. OR @string_idx + 1, if no string "trailer" exists. @@ -2320,7 +2319,7 @@ def _next_state(self, leaf: Leaf) -> bool: MUST be the leaf directly following @leaf. Returns: - True iff @leaf is apart of the string's trailer. + True iff @leaf is a part of the string's trailer. """ # We ignore empty LPAR or RPAR leaves. if is_empty_par(leaf): From c80685f36183f146f831a5737510cf105f947745 Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Thu, 28 Dec 2023 00:24:25 -0600 Subject: [PATCH 03/68] Treat walruses like other binary operators in subscripts (#4109) Fixes #4078 --- CHANGES.md | 3 +++ src/black/lines.py | 9 ++++++++- tests/data/cases/preview_pep_572.py | 4 ++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 526cbd12123..1444463050f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,9 @@ +- Fix bug where spaces were not added around parenthesized walruses in subscripts, + unlike other binary operators (#4109) + ### Configuration diff --git a/src/black/lines.py b/src/black/lines.py index 0cd4189a778..d153b8c2e1b 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -446,8 +446,15 @@ def is_complex_subscript(self, leaf: Leaf) -> bool: if subscript_start.type == syms.subscriptlist: subscript_start = child_towards(subscript_start, leaf) + + # When this is moved out of preview, add syms.namedexpr_test directly to + # TEST_DESCENDANTS in nodes.py + if Preview.walrus_subscript in self.mode: + test_decendants = TEST_DESCENDANTS | {syms.namedexpr_test} + else: + test_decendants = TEST_DESCENDANTS return subscript_start is not None and any( - n.type in TEST_DESCENDANTS for n in subscript_start.pre_order() + n.type in test_decendants for n in subscript_start.pre_order() ) def enumerate_with_length( diff --git a/tests/data/cases/preview_pep_572.py b/tests/data/cases/preview_pep_572.py index 8e801ff6cdc..75ad0cc4176 100644 --- a/tests/data/cases/preview_pep_572.py +++ b/tests/data/cases/preview_pep_572.py @@ -3,5 +3,5 @@ x[:(a:=0)] # output -x[(a := 0):] -x[:(a := 0)] +x[(a := 0) :] +x[: (a := 0)] From bf6cabc8049cbdf4d0b8af33134317a0190a614f Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 27 Dec 2023 22:24:57 -0800 Subject: [PATCH 04/68] Do not round cache mtimes (#4128) Fixes #4116 This logic was introduced in #3821, I believe as a result of copying logic inside mypy that I think isn't relevant to Black --- CHANGES.md | 2 ++ src/black/cache.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1444463050f..2389f6d39fd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,6 +21,8 @@ +- Fix cache mtime logic that resulted in false positive cache hits (#4128) + ### Packaging diff --git a/src/black/cache.py b/src/black/cache.py index c844c37b6f8..cfdbc21e92a 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -101,7 +101,7 @@ def is_changed(self, source: Path) -> bool: st = res_src.stat() if st.st_size != old.st_size: return True - if int(st.st_mtime) != int(old.st_mtime): + if st.st_mtime != old.st_mtime: new_hash = Cache.hash_digest(res_src) if new_hash != old.hash: return True From db9c592967b976a16eccd500f3e2676cfff7f29d Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 27 Dec 2023 22:59:30 -0800 Subject: [PATCH 05/68] Unify docstring detection (#4095) Co-authored-by: hauntsaninja --- CHANGES.md | 1 + src/black/linegen.py | 4 ++-- src/black/lines.py | 15 +++++++++++---- src/black/mode.py | 1 + src/black/nodes.py | 12 +++++++++++- src/black/strings.py | 2 ++ tests/data/cases/module_docstring_2.py | 2 ++ .../preview_no_blank_line_before_docstring.py | 7 +++++++ 8 files changed, 37 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2389f6d39fd..a6587cc5ceb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,7 @@ +- Format module docstrings the same as class and function docstrings (#4095) - Fix bug where spaces were not added around parenthesized walruses in subscripts, unlike other binary operators (#4109) diff --git a/src/black/linegen.py b/src/black/linegen.py index 0fd4a8d9c96..0972cf432e1 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -424,7 +424,7 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: if Preview.hex_codes_in_unicode_sequences in self.mode: normalize_unicode_escape_sequences(leaf) - if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value): + if is_docstring(leaf, self.mode) and not re.search(r"\\\s*\n", leaf.value): # We're ignoring docstrings with backslash newline escapes because changing # indentation of those changes the AST representation of the code. if self.mode.string_normalization: @@ -477,7 +477,7 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: quote = quote_char * quote_len # It's invalid to put closing single-character quotes on a new line. - if self.mode and quote_len == 3: + if quote_len == 3: # We need to find the length of the last line of the docstring # to find if we can add the closing quotes to the line without # exceeding the maximum line length. diff --git a/src/black/lines.py b/src/black/lines.py index d153b8c2e1b..8d02267a85b 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -196,7 +196,7 @@ def is_class_paren_empty(self) -> bool: ) @property - def is_triple_quoted_string(self) -> bool: + def _is_triple_quoted_string(self) -> bool: """Is the line a triple quoted string?""" if not self or self.leaves[0].type != token.STRING: return False @@ -209,6 +209,13 @@ def is_triple_quoted_string(self) -> bool: return True return False + @property + def is_docstring(self) -> bool: + """Is the line a docstring?""" + if Preview.unify_docstring_detection not in self.mode: + return self._is_triple_quoted_string + return bool(self) and is_docstring(self.leaves[0], self.mode) + @property def is_chained_assignment(self) -> bool: """Is the line a chained assignment""" @@ -583,7 +590,7 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: and self.previous_block and self.previous_block.previous_block is None and len(self.previous_block.original_line.leaves) == 1 - and self.previous_block.original_line.is_triple_quoted_string + and self.previous_block.original_line.is_docstring and not (current_line.is_class or current_line.is_def) ): before = 1 @@ -690,7 +697,7 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: if ( self.previous_line and self.previous_line.is_class - and current_line.is_triple_quoted_string + and current_line.is_docstring ): if Preview.no_blank_line_before_class_docstring in current_line.mode: return 0, 1 @@ -701,7 +708,7 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: is_empty_first_line_ok = ( Preview.allow_empty_first_line_in_block in current_line.mode and ( - not is_docstring(current_line.leaves[0]) + not is_docstring(current_line.leaves[0], current_line.mode) or ( self.previous_line and self.previous_line.leaves[0] diff --git a/src/black/mode.py b/src/black/mode.py index 38b861e39ca..466b78228fc 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -195,6 +195,7 @@ class Preview(Enum): single_line_format_skip_with_multiple_comments = auto() long_case_block_line_splitting = auto() allow_form_feeds = auto() + unify_docstring_detection = auto() respect_east_asian_width = auto() diff --git a/src/black/nodes.py b/src/black/nodes.py index a4f555b4032..8e0f27e3ded 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -531,7 +531,7 @@ def is_arith_like(node: LN) -> bool: } -def is_docstring(leaf: Leaf) -> bool: +def is_docstring(leaf: Leaf, mode: Mode) -> bool: if leaf.type != token.STRING: return False @@ -539,6 +539,16 @@ def is_docstring(leaf: Leaf) -> bool: if set(prefix).intersection("bBfF"): return False + if ( + Preview.unify_docstring_detection in mode + and leaf.parent + and leaf.parent.type == syms.simple_stmt + and not leaf.parent.prev_sibling + and leaf.parent.parent + and leaf.parent.parent.type == syms.file_input + ): + return True + if prev_siblings_are( leaf.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt] ): diff --git a/src/black/strings.py b/src/black/strings.py index 0d30f09ed11..0e0f968824b 100644 --- a/src/black/strings.py +++ b/src/black/strings.py @@ -63,6 +63,8 @@ def lines_with_leading_tabs_expanded(s: str) -> List[str]: ) else: lines.append(line) + if s.endswith("\n"): + lines.append("") return lines diff --git a/tests/data/cases/module_docstring_2.py b/tests/data/cases/module_docstring_2.py index e1f81b4d76b..1cc9aea9aea 100644 --- a/tests/data/cases/module_docstring_2.py +++ b/tests/data/cases/module_docstring_2.py @@ -1,6 +1,7 @@ # flags: --preview """I am a very helpful module docstring. +With trailing spaces: Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, @@ -38,6 +39,7 @@ # output """I am a very helpful module docstring. +With trailing spaces: Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, diff --git a/tests/data/cases/preview_no_blank_line_before_docstring.py b/tests/data/cases/preview_no_blank_line_before_docstring.py index 303035a7efb..faeaa1e46e4 100644 --- a/tests/data/cases/preview_no_blank_line_before_docstring.py +++ b/tests/data/cases/preview_no_blank_line_before_docstring.py @@ -29,6 +29,9 @@ class MultilineDocstringsAsWell: and on so many lines... """ +class SingleQuotedDocstring: + + "I'm a docstring but I don't even get triple quotes." # output @@ -57,3 +60,7 @@ class MultilineDocstringsAsWell: and on so many lines... """ + + +class SingleQuotedDocstring: + "I'm a docstring but I don't even get triple quotes." From c35924663cff4f696f9bb91ca9c7775487d95ac6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 15:12:18 -0800 Subject: [PATCH 06/68] [pre-commit.ci] pre-commit autoupdate (#4139) --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2896489d724..13479565527 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: additional_dependencies: *version_check_dependencies - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort @@ -39,7 +39,7 @@ repos: exclude: ^src/blib2to3/ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.7.1 + rev: v1.8.0 hooks: - id: mypy exclude: ^docs/conf.py @@ -58,13 +58,13 @@ repos: - hypothesmith - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.3 + rev: v4.0.0-alpha.8 hooks: - id: prettier exclude: \.github/workflows/diff_shades\.yml - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace From fe3376141c333271d3c64d7fa0e433652e2b48ff Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 1 Jan 2024 15:46:09 -0800 Subject: [PATCH 07/68] Allow empty lines at beginnings of more blocks (#4130) Fixes #4043, fixes #619 These include nested functions and methods. I think the nested function case quite clearly improves readability. I think the method case improves consistency, adherence to PEP 8 and resolves a point of contention. --- CHANGES.md | 2 ++ src/black/lines.py | 5 ++++- tests/data/cases/class_blank_parentheses.py | 1 + .../cases/preview_allow_empty_first_line.py | 19 +++++++++++++++++++ tests/data/cases/preview_form_feeds.py | 1 + 5 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index a6587cc5ceb..fca88612afe 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,8 @@ - Format module docstrings the same as class and function docstrings (#4095) - Fix bug where spaces were not added around parenthesized walruses in subscripts, unlike other binary operators (#4109) +- Address a missing case in the change to allow empty lines at the beginning of all + blocks, except immediately before a docstring (#4130) ### Configuration diff --git a/src/black/lines.py b/src/black/lines.py index 8d02267a85b..4d4f47a44e8 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -745,7 +745,10 @@ def _maybe_empty_lines_for_class_or_def( # noqa: C901 if self.previous_line.depth < current_line.depth and ( self.previous_line.is_class or self.previous_line.is_def ): - return 0, 0 + if self.mode.is_pyi or not Preview.allow_empty_first_line_in_block: + return 0, 0 + else: + return 1 if user_had_newline else 0, 0 comment_to_add_newlines: Optional[LinesBlock] = None if ( diff --git a/tests/data/cases/class_blank_parentheses.py b/tests/data/cases/class_blank_parentheses.py index 1a5721a2889..3c460d9bd79 100644 --- a/tests/data/cases/class_blank_parentheses.py +++ b/tests/data/cases/class_blank_parentheses.py @@ -39,6 +39,7 @@ def test_func(self): class ClassWithEmptyFunc(object): + def func_with_blank_parentheses(): return 5 diff --git a/tests/data/cases/preview_allow_empty_first_line.py b/tests/data/cases/preview_allow_empty_first_line.py index 3e14fa15250..daf78344ad7 100644 --- a/tests/data/cases/preview_allow_empty_first_line.py +++ b/tests/data/cases/preview_allow_empty_first_line.py @@ -62,6 +62,15 @@ def method(self): pass + +def top_level( + a: int, + b: str, +) -> Whatever[Generic, Something]: + + def nested(x: int) -> int: + pass + # output def foo(): @@ -123,6 +132,16 @@ def quux(): class Cls: + def method(self): pass + + +def top_level( + a: int, + b: str, +) -> Whatever[Generic, Something]: + + def nested(x: int) -> int: + pass diff --git a/tests/data/cases/preview_form_feeds.py b/tests/data/cases/preview_form_feeds.py index c236f177a95..dc3bd6cfe2e 100644 --- a/tests/data/cases/preview_form_feeds.py +++ b/tests/data/cases/preview_form_feeds.py @@ -203,6 +203,7 @@ def bar(a=1, b: bool = False): class Baz: + def __init__(self): pass From b9ad4da2e81f6ec66d292b85f284889211e052b4 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 1 Jan 2024 16:55:25 -0800 Subject: [PATCH 08/68] Revert "confine pre-commit to stages (#3940)" (#4137) This reverts commit 7686989fc89aad5ea235a34977ebf8c81c26c4eb. --- .pre-commit-hooks.yaml | 2 -- CHANGES.md | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 54a03efe7a1..a1ff41fded8 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -4,7 +4,6 @@ name: black description: "Black: The uncompromising Python code formatter" entry: black - stages: [pre-commit, pre-merge-commit, pre-push, manual] language: python minimum_pre_commit_version: 2.9.2 require_serial: true @@ -14,7 +13,6 @@ description: "Black: The uncompromising Python code formatter (with Jupyter Notebook support)" entry: black - stages: [pre-commit, pre-merge-commit, pre-push, manual] language: python minimum_pre_commit_version: 2.9.2 require_serial: true diff --git a/CHANGES.md b/CHANGES.md index fca88612afe..360319ac964 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -50,6 +50,9 @@ +- Revert the change to run Black's pre-commit integration only on specific git hooks + (#3940) for better compatibility with older versions of pre-commit (#4137) + ### Documentation +- Fix comment handling when parenthesising conditional expressions (#4134) - Format module docstrings the same as class and function docstrings (#4095) - Fix bug where spaces were not added around parenthesized walruses in subscripts, unlike other binary operators (#4109) diff --git a/src/black/linegen.py b/src/black/linegen.py index 574c89b880c..4d468ce0f2e 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -170,8 +170,12 @@ def visit_test(self, node: Node) -> Iterator[Line]: ) if not already_parenthesized: + # Similar to logic in wrap_in_parentheses lpar = Leaf(token.LPAR, "") rpar = Leaf(token.RPAR, "") + prefix = node.prefix + node.prefix = "" + lpar.prefix = prefix node.insert_child(0, lpar) node.append_child(rpar) diff --git a/tests/data/cases/conditional_expression.py b/tests/data/cases/conditional_expression.py index c30cd76c791..76251bd9318 100644 --- a/tests/data/cases/conditional_expression.py +++ b/tests/data/cases/conditional_expression.py @@ -67,6 +67,28 @@ def something(): else ValuesListIterable ) + +def foo(wait: bool = True): + # This comment is two + # lines long + + # This is only one + time.sleep(1) if wait else None + time.sleep(1) if wait else None + + # With newline above + time.sleep(1) if wait else None + # Without newline above + time.sleep(1) if wait else None + + +a = "".join( + ( + "", # comment + "" if True else "", + ) +) + # output long_kwargs_single_line = my_function( @@ -159,3 +181,23 @@ def something(): if named else FlatValuesListIterable if flat else ValuesListIterable ) + + +def foo(wait: bool = True): + # This comment is two + # lines long + + # This is only one + time.sleep(1) if wait else None + time.sleep(1) if wait else None + + # With newline above + time.sleep(1) if wait else None + # Without newline above + time.sleep(1) if wait else None + + +a = "".join(( + "", # comment + "" if True else "", +)) From e11eaf2f44d3db5713fb99bdec966ba974b60c8c Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 1 Jan 2024 20:14:57 -0800 Subject: [PATCH 13/68] Make `blank_line_after_nested_stub_class` work for methods (#4141) Fixes #4113 Authored by dhruvmanila --- CHANGES.md | 1 + src/black/lines.py | 8 ++++---- tests/data/cases/nested_stub.py | 27 ++++++++++++++++++++++++++- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3dc0c87f89a..8fb8677dd77 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,6 +21,7 @@ - Remove empty lines before docstrings in async functions (#4132) - Address a missing case in the change to allow empty lines at the beginning of all blocks, except immediately before a docstring (#4130) +- For stubs, fix logic to enforce empty line after nested classes with bodies (#4141) ### Configuration diff --git a/src/black/lines.py b/src/black/lines.py index b544c5e0035..9eb5785da57 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -640,15 +640,15 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: if previous_def is not None: assert self.previous_line is not None if self.mode.is_pyi: - if depth and not current_line.is_def and self.previous_line.is_def: - # Empty lines between attributes and methods should be preserved. - before = 1 if user_had_newline else 0 - elif ( + if ( Preview.blank_line_after_nested_stub_class in self.mode and previous_def.is_class and not previous_def.is_stub_class ): before = 1 + elif depth and not current_line.is_def and self.previous_line.is_def: + # Empty lines between attributes and methods should be preserved. + before = 1 if user_had_newline else 0 elif depth: before = 0 else: diff --git a/tests/data/cases/nested_stub.py b/tests/data/cases/nested_stub.py index b81549ec115..ef13c588ce6 100644 --- a/tests/data/cases/nested_stub.py +++ b/tests/data/cases/nested_stub.py @@ -18,6 +18,18 @@ def function_definition(self): ... assignment = 1 def f2(self) -> str: ... + +class TopLevel: + class Nested1: + foo: int + def bar(self): ... + field = 1 + + class Nested2: + def bar(self): ... + foo: int + field = 1 + # output import sys @@ -41,4 +53,17 @@ def f1(self) -> str: ... def function_definition(self): ... assignment = 1 - def f2(self) -> str: ... \ No newline at end of file + def f2(self) -> str: ... + +class TopLevel: + class Nested1: + foo: int + def bar(self): ... + + field = 1 + + class Nested2: + def bar(self): ... + foo: int + + field = 1 From b7c3a9fedd4cfcc6a6a88aacc7b0f599b63d4716 Mon Sep 17 00:00:00 2001 From: Dragorn421 Date: Thu, 11 Jan 2024 16:46:17 +0100 Subject: [PATCH 14/68] Docs: Add note on `--exclude` about possibly verbose regex (#4145) Co-authored-by: Jelle Zijlstra --- docs/usage_and_configuration/the_basics.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 4f9856c6a47..b541f07907c 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -268,6 +268,11 @@ recursive searches. An empty value means no paths are excluded. Use forward slas directories on all platforms (Windows, too). By default, Black also ignores all paths listed in `.gitignore`. Changing this value will override all default exclusions. +If the regular expression contains newlines, it is treated as a +[verbose regular expression](https://docs.python.org/3/library/re.html#re.VERBOSE). This +is typically useful when setting these options in a `pyproject.toml` configuration file; +see [Configuration format](#configuration-format) for more information. + #### `--extend-exclude` Like `--exclude`, but adds additional files and directories on top of the default values From 9a331d606f3fd60cac19bfbfc3f98cbe8be2517d Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Wed, 17 Jan 2024 13:04:15 -0600 Subject: [PATCH 15/68] fix: Don't allow unparenthesizing walruses (#4155) Signed-off-by: RedGuy12 <61329810+RedGuy12@users.noreply.github.com> Signed-off-by: RedGuy12 --- CHANGES.md | 1 + src/black/linegen.py | 6 +++++- tests/data/cases/walrus_in_dict.py | 7 +++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/data/cases/walrus_in_dict.py diff --git a/CHANGES.md b/CHANGES.md index 8fb8677dd77..2bd58ed49ff 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -22,6 +22,7 @@ - Address a missing case in the change to allow empty lines at the beginning of all blocks, except immediately before a docstring (#4130) - For stubs, fix logic to enforce empty line after nested classes with bodies (#4141) +- Fix crash when using a walrus in a dictionary (#4155) ### Configuration diff --git a/src/black/linegen.py b/src/black/linegen.py index 4d468ce0f2e..9a3eb0ce73f 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -242,7 +242,11 @@ def visit_dictsetmaker(self, node: Node) -> Iterator[Line]: if i == 0: continue if node.children[i - 1].type == token.COLON: - if child.type == syms.atom and child.children[0].type == token.LPAR: + if ( + child.type == syms.atom + and child.children[0].type == token.LPAR + and not is_walrus_assignment(child) + ): if maybe_make_parens_invisible_in_atom( child, parent=node, diff --git a/tests/data/cases/walrus_in_dict.py b/tests/data/cases/walrus_in_dict.py new file mode 100644 index 00000000000..c33eecd84a6 --- /dev/null +++ b/tests/data/cases/walrus_in_dict.py @@ -0,0 +1,7 @@ +# flags: --preview +{ + "is_update": (up := commit.hash in update_hashes) +} + +# output +{"is_update": (up := commit.hash in update_hashes)} From 7f60f3dbd7d2d36011fbae6c140b35802932952b Mon Sep 17 00:00:00 2001 From: Kevin Paulson Date: Fri, 19 Jan 2024 18:54:32 -0500 Subject: [PATCH 16/68] Update using_black_with_other_tools.md to ensure flake8 configuration examples are consistant (#4157) --- docs/guides/using_black_with_other_tools.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/guides/using_black_with_other_tools.md b/docs/guides/using_black_with_other_tools.md index 22c641a7420..e642a1aef33 100644 --- a/docs/guides/using_black_with_other_tools.md +++ b/docs/guides/using_black_with_other_tools.md @@ -145,7 +145,7 @@ There are a few deviations that cause incompatibilities with _Black_. ``` max-line-length = 88 -extend-ignore = E203 +extend-ignore = E203, E704 ``` #### Why those options above? @@ -184,7 +184,7 @@ extend-ignore = E203, E704 ```ini [flake8] max-line-length = 88 -extend-ignore = E203 +extend-ignore = E203, E704 ``` @@ -195,7 +195,7 @@ extend-ignore = E203 ```ini [flake8] max-line-length = 88 -extend-ignore = E203 +extend-ignore = E203, E704 ``` From 995e4ada14d63a9bec39c5fc83275d0e49742618 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 19 Jan 2024 17:13:26 -0800 Subject: [PATCH 17/68] Fix unnecessary nesting when wrapping long dict (#4135) Fixes #4129 --- CHANGES.md | 1 + src/black/linegen.py | 7 ++-- tests/data/cases/preview_long_dict_values.py | 38 ++++++++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2bd58ed49ff..1e75fb58563 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -23,6 +23,7 @@ blocks, except immediately before a docstring (#4130) - For stubs, fix logic to enforce empty line after nested classes with bodies (#4141) - Fix crash when using a walrus in a dictionary (#4155) +- Fix unnecessary parentheses when wrapping long dicts (#4135) ### Configuration diff --git a/src/black/linegen.py b/src/black/linegen.py index 9a3eb0ce73f..dd296eb801d 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -244,15 +244,14 @@ def visit_dictsetmaker(self, node: Node) -> Iterator[Line]: if node.children[i - 1].type == token.COLON: if ( child.type == syms.atom - and child.children[0].type == token.LPAR + and child.children[0].type in OPENING_BRACKETS and not is_walrus_assignment(child) ): - if maybe_make_parens_invisible_in_atom( + maybe_make_parens_invisible_in_atom( child, parent=node, remove_brackets_around_comma=False, - ): - wrap_in_parentheses(node, child, visible=False) + ) else: wrap_in_parentheses(node, child, visible=False) yield from self.visit_default(node) diff --git a/tests/data/cases/preview_long_dict_values.py b/tests/data/cases/preview_long_dict_values.py index fbbacd13d1d..54da76038dc 100644 --- a/tests/data/cases/preview_long_dict_values.py +++ b/tests/data/cases/preview_long_dict_values.py @@ -37,6 +37,26 @@ } +class Random: + def func(): + random_service.status.active_states.inactive = ( + make_new_top_level_state_from_dict( + { + "topLevelBase": { + "secondaryBase": { + "timestamp": 1234, + "latitude": 1, + "longitude": 2, + "actionTimestamp": Timestamp( + seconds=1530584000, nanos=0 + ).ToJsonString(), + } + }, + } + ) + ) + + # output @@ -89,3 +109,21 @@ } ), } + + +class Random: + def func(): + random_service.status.active_states.inactive = ( + make_new_top_level_state_from_dict({ + "topLevelBase": { + "secondaryBase": { + "timestamp": 1234, + "latitude": 1, + "longitude": 2, + "actionTimestamp": ( + Timestamp(seconds=1530584000, nanos=0).ToJsonString() + ), + } + }, + }) + ) From 6f3fb78444655f883780dcc19349226833c677c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 09:22:56 -0800 Subject: [PATCH 18/68] Bump actions/cache from 3 to 4 (#4162) Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/diff_shades.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/diff_shades.yml b/.github/workflows/diff_shades.yml index 8d8be2550b0..0e1aab00e34 100644 --- a/.github/workflows/diff_shades.yml +++ b/.github/workflows/diff_shades.yml @@ -72,7 +72,7 @@ jobs: - name: Attempt to use cached baseline analysis id: baseline-cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ matrix.baseline-analysis }} key: ${{ matrix.baseline-cache-key }} From 8fe602b1fa91dc6db682d1dba79a8a7341597271 Mon Sep 17 00:00:00 2001 From: Daniel Krzeminski Date: Mon, 22 Jan 2024 11:46:57 -0600 Subject: [PATCH 19/68] fix pathlib exception handling with symlinks (#4161) Fixes #4077 --- CHANGES.md | 2 ++ src/black/__init__.py | 6 +++++- src/black/files.py | 24 ++++++++++++++++-------- tests/test_black.py | 14 ++++++++++++++ 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1e75fb58563..f29834a3f7f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -29,6 +29,8 @@ +- Fix symlink handling, properly catch and ignore symlinks that point outside of root + (#4161) - Fix cache mtime logic that resulted in false positive cache hits (#4128) ### Packaging diff --git a/src/black/__init__.py b/src/black/__init__.py index 735ba713b8f..e3cbaab5f1d 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -49,6 +49,7 @@ find_user_pyproject_toml, gen_python_files, get_gitignore, + get_root_relative_path, normalize_path_maybe_ignore, parse_pyproject_toml, path_is_excluded, @@ -700,7 +701,10 @@ def get_sources( # Compare the logic here to the logic in `gen_python_files`. if is_stdin or path.is_file(): - root_relative_path = path.absolute().relative_to(root).as_posix() + root_relative_path = get_root_relative_path(path, root, report) + + if root_relative_path is None: + continue root_relative_path = "/" + root_relative_path diff --git a/src/black/files.py b/src/black/files.py index 858303ca1a3..65951efdbe8 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -259,14 +259,7 @@ def normalize_path_maybe_ignore( try: abspath = path if path.is_absolute() else Path.cwd() / path normalized_path = abspath.resolve() - try: - root_relative_path = normalized_path.relative_to(root).as_posix() - except ValueError: - if report: - report.path_ignored( - path, f"is a symbolic link that points outside {root}" - ) - return None + root_relative_path = get_root_relative_path(normalized_path, root, report) except OSError as e: if report: @@ -276,6 +269,21 @@ def normalize_path_maybe_ignore( return root_relative_path +def get_root_relative_path( + path: Path, + root: Path, + report: Optional[Report] = None, +) -> Optional[str]: + """Returns the file path relative to the 'root' directory""" + try: + root_relative_path = path.absolute().relative_to(root).as_posix() + except ValueError: + if report: + report.path_ignored(path, f"is a symbolic link that points outside {root}") + return None + return root_relative_path + + def _path_is_ignored( root_relative_path: str, root: Path, diff --git a/tests/test_black.py b/tests/test_black.py index 0af5fd2a1f4..2b5fab5d28d 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2592,6 +2592,20 @@ def test_symlinks(self) -> None: outside_root_symlink.resolve.assert_called_once() ignored_symlink.resolve.assert_not_called() + def test_get_sources_with_stdin_symlink_outside_root( + self, + ) -> None: + path = THIS_DIR / "data" / "include_exclude_tests" + stdin_filename = str(path / "b/exclude/a.py") + outside_root_symlink = Path("/target_directory/a.py") + with patch("pathlib.Path.resolve", return_value=outside_root_symlink): + assert_collected_sources( + root=Path("target_directory/"), + src=["-"], + expected=[], + stdin_filename=stdin_filename, + ) + @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin(self) -> None: src = ["-"] From 59b9d858a30de56801e84c31f57b53337c61647c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 24 Jan 2024 17:06:14 -0800 Subject: [PATCH 20/68] Create the 2024 stable style (#4106) --- CHANGES.md | 40 +++++++- src/black/comments.py | 32 +++---- src/black/linegen.py | 95 +++++-------------- src/black/lines.py | 63 +++--------- src/black/mode.py | 25 +---- src/black/nodes.py | 14 +-- ...irst_line.py => allow_empty_first_line.py} | 1 - ...{preview_async_stmts.py => async_stmts.py} | 1 - tests/data/cases/comments5.py | 9 +- tests/data/cases/conditional_expression.py | 11 ++- ..._managers_38.py => context_managers_38.py} | 2 +- ..._managers_39.py => context_managers_39.py} | 2 +- ....py => context_managers_autodetect_310.py} | 2 +- ....py => context_managers_autodetect_311.py} | 2 +- ...8.py => context_managers_autodetect_38.py} | 1 - ...9.py => context_managers_autodetect_39.py} | 2 +- ...mentations.py => dummy_implementations.py} | 1 - tests/data/cases/empty_lines.py | 1 + tests/data/cases/fmtonoff.py | 8 +- tests/data/cases/fmtonoff5.py | 3 +- .../{preview_form_feeds.py => form_feeds.py} | 1 - tests/data/cases/function.py | 5 +- ...r_match.py => keep_newline_after_match.py} | 6 ++ .../data/cases/long_strings_flag_disabled.py | 16 ++-- tests/data/cases/module_docstring_1.py | 1 - tests/data/cases/module_docstring_2.py | 4 +- tests/data/cases/module_docstring_3.py | 1 - tests/data/cases/module_docstring_4.py | 1 - .../module_docstring_followed_by_class.py | 1 - .../module_docstring_followed_by_function.py | 1 - tests/data/cases/nested_stub.py | 2 +- ...g.py => no_blank_line_before_docstring.py} | 2 +- ...ching_long.py => pattern_matching_long.py} | 2 +- ....py => pattern_matching_trailing_comma.py} | 2 +- .../cases/pep604_union_types_line_breaks.py | 2 +- tests/data/cases/pep_572_py310.py | 6 +- tests/data/cases/pep_572_remove_parens.py | 8 +- .../{preview_pep_572.py => pep_572_slices.py} | 1 - ...nt_precedence.py => percent_precedence.py} | 5 +- ...op_spacing.py => power_op_spacing_long.py} | 1 - ...refer_rhs_split.py => prefer_rhs_split.py} | 1 - tests/data/cases/py310_pep572.py | 2 +- tests/data/cases/python39.py | 9 +- tests/data/cases/raw_docstring.py | 2 +- ... raw_docstring_no_string_normalization.py} | 2 +- .../remove_newline_after_code_block_open.py | 79 +++++++++------ .../data/cases/return_annotation_brackets.py | 34 +++---- ...ine_format_skip_with_multiple_comments.py} | 1 - ...ew_trailing_comma.py => trailing_comma.py} | 1 - tests/data/cases/walrus_in_dict.py | 2 + 50 files changed, 222 insertions(+), 294 deletions(-) rename tests/data/cases/{preview_allow_empty_first_line.py => allow_empty_first_line.py} (98%) rename tests/data/cases/{preview_async_stmts.py => async_stmts.py} (93%) rename tests/data/cases/{preview_context_managers_38.py => context_managers_38.py} (96%) rename tests/data/cases/{preview_context_managers_39.py => context_managers_39.py} (98%) rename tests/data/cases/{preview_context_managers_autodetect_310.py => context_managers_autodetect_310.py} (93%) rename tests/data/cases/{preview_context_managers_autodetect_311.py => context_managers_autodetect_311.py} (92%) rename tests/data/cases/{preview_context_managers_autodetect_38.py => context_managers_autodetect_38.py} (98%) rename tests/data/cases/{preview_context_managers_autodetect_39.py => context_managers_autodetect_39.py} (93%) rename tests/data/cases/{preview_dummy_implementations.py => dummy_implementations.py} (99%) rename tests/data/cases/{preview_form_feeds.py => form_feeds.py} (99%) rename tests/data/cases/{remove_newline_after_match.py => keep_newline_after_match.py} (98%) rename tests/data/cases/{preview_no_blank_line_before_docstring.py => no_blank_line_before_docstring.py} (98%) rename tests/data/cases/{preview_pattern_matching_long.py => pattern_matching_long.py} (94%) rename tests/data/cases/{preview_pattern_matching_trailing_comma.py => pattern_matching_trailing_comma.py} (92%) rename tests/data/cases/{preview_pep_572.py => pep_572_slices.py} (75%) rename tests/data/cases/{preview_percent_precedence.py => percent_precedence.py} (91%) rename tests/data/cases/{preview_power_op_spacing.py => power_op_spacing_long.py} (99%) rename tests/data/cases/{preview_prefer_rhs_split.py => prefer_rhs_split.py} (99%) rename tests/data/cases/{preview_docstring_no_string_normalization.py => raw_docstring_no_string_normalization.py} (88%) rename tests/data/cases/{preview_single_line_format_skip_with_multiple_comments.py => single_line_format_skip_with_multiple_comments.py} (97%) rename tests/data/cases/{preview_trailing_comma.py => trailing_comma.py} (97%) diff --git a/CHANGES.md b/CHANGES.md index f29834a3f7f..0e2974d706e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,22 +6,54 @@ +This release introduces the new 2024 stable style (#4106), stabilizing the following +changes: + +- Add parentheses around `if`-`else` expressions (#2278) +- Dummy class and function implementations consisting only of `...` are formatted more + compactly (#3796) +- If an assignment statement is too long, we now prefer splitting on the right-hand side + (#3368) +- Hex codes in Unicode escape sequences are now standardized to lowercase (#2916) +- Allow empty first lines at the beginning of most blocks (#3967, #4061) +- Add parentheses around long type annotations (#3899) +- Standardize on a single newline after module docstrings (#3932) +- Fix incorrect magic trailing comma handling in return types (#3916) +- Remove blank lines before class docstrings (#3692) +- Wrap multiple context managers in parentheses if combined in a single `with` statement + (#3489) +- Fix bug in line length calculations for power operations (#3942) +- Add trailing commas to collection literals even if there's a comment after the last + entry (#3393) +- When using `--skip-magic-trailing-comma` or `-C`, trailing commas are stripped from + subscript expressions with more than 1 element (#3209) +- Add extra blank lines in stubs in a few cases (#3564, #3862) +- Accept raw strings as docstrings (#3947) +- Split long lines in case blocks (#4024) +- Stop removing spaces from walrus operators within subscripts (#3823) +- Fix incorrect formatting of certain async statements (#3609) +- Allow combining `# fmt: skip` with other comments (#3959) + ### Stable style -### Preview style - - +Several bug fixes were made in features that are moved to the stable style in this +release: - Fix comment handling when parenthesising conditional expressions (#4134) -- Format module docstrings the same as class and function docstrings (#4095) - Fix bug where spaces were not added around parenthesized walruses in subscripts, unlike other binary operators (#4109) - Remove empty lines before docstrings in async functions (#4132) - Address a missing case in the change to allow empty lines at the beginning of all blocks, except immediately before a docstring (#4130) - For stubs, fix logic to enforce empty line after nested classes with bodies (#4141) + +### Preview style + + + +- Format module docstrings the same as class and function docstrings (#4095) - Fix crash when using a walrus in a dictionary (#4155) - Fix unnecessary parentheses when wrapping long dicts (#4135) diff --git a/src/black/comments.py b/src/black/comments.py index 52bb024a799..910e1b760f0 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -3,7 +3,7 @@ from functools import lru_cache from typing import Collection, Final, Iterator, List, Optional, Tuple, Union -from black.mode import Mode, Preview +from black.mode import Mode from black.nodes import ( CLOSING_BRACKETS, STANDALONE_COMMENT, @@ -390,22 +390,18 @@ def _contains_fmt_skip_comment(comment_line: str, mode: Mode) -> bool: # noqa:XXX # fmt:skip # a nice line <-- multiple comments (Preview) # pylint:XXX; fmt:skip <-- list of comments (; separated, Preview) """ - semantic_comment_blocks = ( - [ - comment_line, - *[ - _COMMENT_PREFIX + comment.strip() - for comment in comment_line.split(_COMMENT_PREFIX)[1:] - ], - *[ - _COMMENT_PREFIX + comment.strip() - for comment in comment_line.strip(_COMMENT_PREFIX).split( - _COMMENT_LIST_SEPARATOR - ) - ], - ] - if Preview.single_line_format_skip_with_multiple_comments in mode - else [comment_line] - ) + semantic_comment_blocks = [ + comment_line, + *[ + _COMMENT_PREFIX + comment.strip() + for comment in comment_line.split(_COMMENT_PREFIX)[1:] + ], + *[ + _COMMENT_PREFIX + comment.strip() + for comment in comment_line.strip(_COMMENT_PREFIX).split( + _COMMENT_LIST_SEPARATOR + ) + ], + ] return any(comment in FMT_SKIP for comment in semantic_comment_blocks) diff --git a/src/black/linegen.py b/src/black/linegen.py index dd296eb801d..a276805f2fe 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -115,10 +115,8 @@ def line(self, indent: int = 0) -> Iterator[Line]: self.current_line.depth += indent return # Line is empty, don't emit. Creating a new one unnecessary. - if ( - Preview.improved_async_statements_handling in self.mode - and len(self.current_line.leaves) == 1 - and is_async_stmt_or_funcdef(self.current_line.leaves[0]) + if len(self.current_line.leaves) == 1 and is_async_stmt_or_funcdef( + self.current_line.leaves[0] ): # Special case for async def/for/with statements. `visit_async_stmt` # adds an `ASYNC` leaf then visits the child def/for/with statement @@ -164,20 +162,19 @@ def visit_default(self, node: LN) -> Iterator[Line]: def visit_test(self, node: Node) -> Iterator[Line]: """Visit an `x if y else z` test""" - if Preview.parenthesize_conditional_expressions in self.mode: - already_parenthesized = ( - node.prev_sibling and node.prev_sibling.type == token.LPAR - ) + already_parenthesized = ( + node.prev_sibling and node.prev_sibling.type == token.LPAR + ) - if not already_parenthesized: - # Similar to logic in wrap_in_parentheses - lpar = Leaf(token.LPAR, "") - rpar = Leaf(token.RPAR, "") - prefix = node.prefix - node.prefix = "" - lpar.prefix = prefix - node.insert_child(0, lpar) - node.append_child(rpar) + if not already_parenthesized: + # Similar to logic in wrap_in_parentheses + lpar = Leaf(token.LPAR, "") + rpar = Leaf(token.RPAR, "") + prefix = node.prefix + node.prefix = "" + lpar.prefix = prefix + node.insert_child(0, lpar) + node.append_child(rpar) yield from self.visit_default(node) @@ -292,9 +289,7 @@ def visit_match_case(self, node: Node) -> Iterator[Line]: def visit_suite(self, node: Node) -> Iterator[Line]: """Visit a suite.""" - if ( - self.mode.is_pyi or Preview.dummy_implementations in self.mode - ) and is_stub_suite(node, self.mode): + if is_stub_suite(node): yield from self.visit(node.children[2]) else: yield from self.visit_default(node) @@ -308,11 +303,7 @@ def visit_simple_stmt(self, node: Node) -> Iterator[Line]: prev_type = child.type if node.parent and node.parent.type in STATEMENT: - if Preview.dummy_implementations in self.mode: - condition = is_parent_function_or_class(node) - else: - condition = self.mode.is_pyi - if condition and is_stub_body(node): + if is_parent_function_or_class(node) and is_stub_body(node): yield from self.visit_default(node) else: yield from self.line(+1) @@ -320,11 +311,7 @@ def visit_simple_stmt(self, node: Node) -> Iterator[Line]: yield from self.line(-1) else: - if ( - not (self.mode.is_pyi or Preview.dummy_implementations in self.mode) - or not node.parent - or not is_stub_suite(node.parent, self.mode) - ): + if not node.parent or not is_stub_suite(node.parent): yield from self.line() yield from self.visit_default(node) @@ -342,11 +329,7 @@ def visit_async_stmt(self, node: Node) -> Iterator[Line]: break internal_stmt = next(children) - if Preview.improved_async_statements_handling in self.mode: - yield from self.visit(internal_stmt) - else: - for child in internal_stmt.children: - yield from self.visit(child) + yield from self.visit(internal_stmt) def visit_decorators(self, node: Node) -> Iterator[Line]: """Visit decorators.""" @@ -420,10 +403,9 @@ def foo(a: int, b: float = 7): ... def foo(a: (int), b: (float) = 7): ... """ - if Preview.parenthesize_long_type_hints in self.mode: - assert len(node.children) == 3 - if maybe_make_parens_invisible_in_atom(node.children[2], parent=node): - wrap_in_parentheses(node, node.children[2], visible=False) + assert len(node.children) == 3 + if maybe_make_parens_invisible_in_atom(node.children[2], parent=node): + wrap_in_parentheses(node, node.children[2], visible=False) yield from self.visit_default(node) @@ -529,13 +511,7 @@ def __post_init__(self) -> None: self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"}) self.visit_classdef = partial(v, keywords={"class"}, parens=Ø) - # When this is moved out of preview, add ":" directly to ASSIGNMENTS in nodes.py - if Preview.parenthesize_long_type_hints in self.mode: - assignments = ASSIGNMENTS | {":"} - else: - assignments = ASSIGNMENTS - self.visit_expr_stmt = partial(v, keywords=Ø, parens=assignments) - + self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS) self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"}) self.visit_import_from = partial(v, keywords=Ø, parens={"import"}) self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"}) @@ -576,9 +552,7 @@ def transform_line( # We need the line string when power operators are hugging to determine if we should # split the line. Default to line_str, if no power operator are present on the line. line_str_hugging_power_ops = ( - (_hugging_power_ops_line_to_string(line, features, mode) or line_str) - if Preview.fix_power_op_line_length in mode - else line_str + _hugging_power_ops_line_to_string(line, features, mode) or line_str ) ll = mode.line_length @@ -688,9 +662,6 @@ def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool: """If a funcdef has a magic trailing comma in the return type, then we should first split the line with rhs to respect the comma. """ - if Preview.respect_magic_trailing_comma_in_return_type not in mode: - return False - return_type_leaves: List[Leaf] = [] in_return_type = False @@ -919,9 +890,6 @@ def _maybe_split_omitting_optional_parens( try: # The RHSResult Omitting Optional Parens. rhs_oop = _first_right_hand_split(line, omit=omit) - prefer_splitting_rhs_mode = ( - Preview.prefer_splitting_right_hand_side_of_assignments in line.mode - ) is_split_right_after_equal = ( len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL ) @@ -937,8 +905,7 @@ def _maybe_split_omitting_optional_parens( ) if ( not ( - prefer_splitting_rhs_mode - and is_split_right_after_equal + is_split_right_after_equal and rhs_head_contains_brackets and rhs_head_short_enough and rhs_head_explode_blocked_by_magic_trailing_comma @@ -1224,11 +1191,7 @@ def append_to_line(leaf: Leaf) -> Iterator[Line]: trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features ) - if ( - Preview.add_trailing_comma_consistently in mode - and last_leaf.type == STANDALONE_COMMENT - and leaf_idx == last_non_comment_leaf - ): + if last_leaf.type == STANDALONE_COMMENT and leaf_idx == last_non_comment_leaf: current_line = _safe_add_trailing_comma( trailing_comma_safe, delimiter_priority, current_line ) @@ -1315,11 +1278,7 @@ def normalize_invisible_parens( # noqa: C901 # Fixes a bug where invisible parens are not properly wrapped around # case blocks. - if ( - isinstance(child, Node) - and child.type == syms.case_block - and Preview.long_case_block_line_splitting in mode - ): + if isinstance(child, Node) and child.type == syms.case_block: normalize_invisible_parens( child, parens_after={"case"}, mode=mode, features=features ) @@ -1374,7 +1333,6 @@ def normalize_invisible_parens( # noqa: C901 and child.next_sibling is not None and child.next_sibling.type == token.COLON and child.value == "case" - and Preview.long_case_block_line_splitting in mode ): # A special patch for "case case:" scenario, the second occurrence # of case will be not parsed as a Python keyword. @@ -1448,7 +1406,6 @@ def _maybe_wrap_cms_in_parens( """ if ( Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features - or Preview.wrap_multiple_context_managers_in_parens not in mode or len(node.children) <= 2 # If it's an atom, it's already wrapped in parens. or node.children[1].type == syms.atom diff --git a/src/black/lines.py b/src/black/lines.py index 9eb5785da57..29f87137614 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -202,9 +202,7 @@ def _is_triple_quoted_string(self) -> bool: value = self.leaves[0].value if value.startswith(('"""', "'''")): return True - if Preview.accept_raw_docstrings in self.mode and value.startswith( - ("r'''", 'r"""', "R'''", 'R"""') - ): + if value.startswith(("r'''", 'r"""', "R'''", 'R"""')): return True return False @@ -450,14 +448,8 @@ def is_complex_subscript(self, leaf: Leaf) -> bool: if subscript_start.type == syms.subscriptlist: subscript_start = child_towards(subscript_start, leaf) - # When this is moved out of preview, add syms.namedexpr_test directly to - # TEST_DESCENDANTS in nodes.py - if Preview.walrus_subscript in self.mode: - test_decendants = TEST_DESCENDANTS | {syms.namedexpr_test} - else: - test_decendants = TEST_DESCENDANTS return subscript_start is not None and any( - n.type in test_decendants for n in subscript_start.pre_order() + n.type in TEST_DESCENDANTS for n in subscript_start.pre_order() ) def enumerate_with_length( @@ -567,8 +559,7 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: lines (two on module-level). """ form_feed = ( - Preview.allow_form_feeds in self.mode - and current_line.depth == 0 + current_line.depth == 0 and bool(current_line.leaves) and "\f\n" in current_line.leaves[0].prefix ) @@ -582,8 +573,7 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: else before - previous_after ) if ( - Preview.module_docstring_newlines in current_line.mode - and self.previous_block + self.previous_block and self.previous_block.previous_block is None and len(self.previous_block.original_line.leaves) == 1 and self.previous_block.original_line.is_docstring @@ -640,11 +630,7 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: if previous_def is not None: assert self.previous_line is not None if self.mode.is_pyi: - if ( - Preview.blank_line_after_nested_stub_class in self.mode - and previous_def.is_class - and not previous_def.is_stub_class - ): + if previous_def.is_class and not previous_def.is_stub_class: before = 1 elif depth and not current_line.is_def and self.previous_line.is_def: # Empty lines between attributes and methods should be preserved. @@ -695,18 +681,12 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: and self.previous_line.is_class and current_line.is_docstring ): - if Preview.no_blank_line_before_class_docstring in current_line.mode: - return 0, 1 - return before, 1 + return 0, 1 # In preview mode, always allow blank lines, except right before a function # docstring - is_empty_first_line_ok = ( - Preview.allow_empty_first_line_in_block in current_line.mode - and ( - not current_line.is_docstring - or (self.previous_line and not self.previous_line.is_def) - ) + is_empty_first_line_ok = not current_line.is_docstring or ( + self.previous_line and not self.previous_line.is_def ) if ( @@ -736,7 +716,7 @@ def _maybe_empty_lines_for_class_or_def( # noqa: C901 if self.previous_line.depth < current_line.depth and ( self.previous_line.is_class or self.previous_line.is_def ): - if self.mode.is_pyi or not Preview.allow_empty_first_line_in_block: + if self.mode.is_pyi: return 0, 0 else: return 1 if user_had_newline else 0, 0 @@ -776,10 +756,7 @@ def _maybe_empty_lines_for_class_or_def( # noqa: C901 # Don't inspect the previous line if it's part of the body of the previous # statement in the same level, we always want a blank line if there's # something with a body preceding. - elif ( - Preview.blank_line_between_nested_and_def_stub_file in current_line.mode - and self.previous_line.depth > current_line.depth - ): + elif self.previous_line.depth > current_line.depth: newlines = 1 elif ( current_line.is_def or current_line.is_decorator @@ -800,11 +777,7 @@ def _maybe_empty_lines_for_class_or_def( # noqa: C901 newlines = 1 if current_line.depth else 2 # If a user has left no space after a dummy implementation, don't insert # new lines. This is useful for instance for @overload or Protocols. - if ( - Preview.dummy_implementations in self.mode - and self.previous_line.is_stub_def - and not user_had_newline - ): + if self.previous_line.is_stub_def and not user_had_newline: newlines = 0 if comment_to_add_newlines is not None: previous_block = comment_to_add_newlines.previous_block @@ -859,11 +832,9 @@ def is_line_short_enough( # noqa: C901 if not line_str: line_str = line_to_string(line) - width = str_width if Preview.respect_east_asian_width in mode else len - if Preview.multiline_string_handling not in mode: return ( - width(line_str) <= mode.line_length + str_width(line_str) <= mode.line_length and "\n" not in line_str # multiline strings and not line.contains_standalone_comments() ) @@ -872,10 +843,10 @@ def is_line_short_enough( # noqa: C901 return False if "\n" not in line_str: # No multiline strings (MLS) present - return width(line_str) <= mode.line_length + return str_width(line_str) <= mode.line_length first, *_, last = line_str.split("\n") - if width(first) > mode.line_length or width(last) > mode.line_length: + if str_width(first) > mode.line_length or str_width(last) > mode.line_length: return False # Traverse the AST to examine the context of the multiline string (MLS), @@ -1015,11 +986,7 @@ def can_omit_invisible_parens( return False if delimiter_count == 1: - if ( - Preview.wrap_multiple_context_managers_in_parens in line.mode - and max_priority == COMMA_PRIORITY - and rhs.head.is_with_or_async_with_stmt - ): + if max_priority == COMMA_PRIORITY and rhs.head.is_with_or_async_with_stmt: # For two context manager with statements, the optional parentheses read # better. In this case, `rhs.body` is the context managers part of # the with statement. `rhs.head` is the `with (` part on the previous diff --git a/src/black/mode.py b/src/black/mode.py index 466b78228fc..1b97f3508ee 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -168,35 +168,14 @@ def supports_feature(target_versions: Set[TargetVersion], feature: Feature) -> b class Preview(Enum): """Individual preview style features.""" - add_trailing_comma_consistently = auto() - blank_line_after_nested_stub_class = auto() - blank_line_between_nested_and_def_stub_file = auto() hex_codes_in_unicode_sequences = auto() - improved_async_statements_handling = auto() - multiline_string_handling = auto() - no_blank_line_before_class_docstring = auto() - prefer_splitting_right_hand_side_of_assignments = auto() # NOTE: string_processing requires wrap_long_dict_values_in_parens # for https://github.com/psf/black/issues/3117 to be fixed. string_processing = auto() - parenthesize_conditional_expressions = auto() - parenthesize_long_type_hints = auto() - respect_magic_trailing_comma_in_return_type = auto() - skip_magic_trailing_comma_in_subscript = auto() - wrap_long_dict_values_in_parens = auto() - wrap_multiple_context_managers_in_parens = auto() - dummy_implementations = auto() - walrus_subscript = auto() - module_docstring_newlines = auto() - accept_raw_docstrings = auto() - fix_power_op_line_length = auto() hug_parens_with_braces_and_square_brackets = auto() - allow_empty_first_line_in_block = auto() - single_line_format_skip_with_multiple_comments = auto() - long_case_block_line_splitting = auto() - allow_form_feeds = auto() unify_docstring_detection = auto() - respect_east_asian_width = auto() + wrap_long_dict_values_in_parens = auto() + multiline_string_handling = auto() class Deprecated(UserWarning): diff --git a/src/black/nodes.py b/src/black/nodes.py index 7ee2df2e061..a8869cba234 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -104,6 +104,7 @@ syms.trailer, syms.term, syms.power, + syms.namedexpr_test, } TYPED_NAMES: Final = {syms.tname, syms.tname_star} ASSIGNMENTS: Final = { @@ -121,6 +122,7 @@ ">>=", "**=", "//=", + ":", } IMPLICIT_TUPLE: Final = {syms.testlist, syms.testlist_star_expr, syms.exprlist} @@ -346,9 +348,7 @@ def whitespace(leaf: Leaf, *, complex_subscript: bool, mode: Mode) -> str: # no return NO - elif Preview.walrus_subscript in mode and ( - t == token.COLONEQUAL or prev.type == token.COLONEQUAL - ): + elif t == token.COLONEQUAL or prev.type == token.COLONEQUAL: return SPACE elif not complex_subscript: @@ -753,13 +753,9 @@ def is_function_or_class(node: Node) -> bool: return node.type in {syms.funcdef, syms.classdef, syms.async_funcdef} -def is_stub_suite(node: Node, mode: Mode) -> bool: +def is_stub_suite(node: Node) -> bool: """Return True if `node` is a suite with a stub body.""" - if ( - node.parent is not None - and Preview.dummy_implementations in mode - and not is_parent_function_or_class(node) - ): + if node.parent is not None and not is_parent_function_or_class(node): return False # If there is a comment, we want to keep it. diff --git a/tests/data/cases/preview_allow_empty_first_line.py b/tests/data/cases/allow_empty_first_line.py similarity index 98% rename from tests/data/cases/preview_allow_empty_first_line.py rename to tests/data/cases/allow_empty_first_line.py index 4269987305d..32a170a97d0 100644 --- a/tests/data/cases/preview_allow_empty_first_line.py +++ b/tests/data/cases/allow_empty_first_line.py @@ -1,4 +1,3 @@ -# flags: --preview def foo(): """ Docstring diff --git a/tests/data/cases/preview_async_stmts.py b/tests/data/cases/async_stmts.py similarity index 93% rename from tests/data/cases/preview_async_stmts.py rename to tests/data/cases/async_stmts.py index 0a7671be5a6..fe9594b2164 100644 --- a/tests/data/cases/preview_async_stmts.py +++ b/tests/data/cases/async_stmts.py @@ -1,4 +1,3 @@ -# flags: --preview async def func() -> (int): return 0 diff --git a/tests/data/cases/comments5.py b/tests/data/cases/comments5.py index bda40619f62..4270d3a09a2 100644 --- a/tests/data/cases/comments5.py +++ b/tests/data/cases/comments5.py @@ -45,8 +45,7 @@ def wat(): @deco2(with_args=True) # leading 3 @deco3 -def decorated1(): - ... +def decorated1(): ... # leading 1 @@ -54,8 +53,7 @@ def decorated1(): # leading 2 @deco2(with_args=True) # leading function comment -def decorated1(): - ... +def decorated1(): ... # Note: this is fixed in @@ -65,8 +63,7 @@ def decorated1(): # This comment should be split from `some_instruction` by two lines but isn't. -def g(): - ... +def g(): ... if __name__ == "__main__": diff --git a/tests/data/cases/conditional_expression.py b/tests/data/cases/conditional_expression.py index 76251bd9318..f65d6fb00e7 100644 --- a/tests/data/cases/conditional_expression.py +++ b/tests/data/cases/conditional_expression.py @@ -1,4 +1,3 @@ -# flags: --preview long_kwargs_single_line = my_function( foo="test, this is a sample value", bar=some_long_value_name_foo_bar_baz if some_boolean_variable else some_fallback_value_foo_bar_baz, @@ -197,7 +196,9 @@ def foo(wait: bool = True): time.sleep(1) if wait else None -a = "".join(( - "", # comment - "" if True else "", -)) +a = "".join( + ( + "", # comment + "" if True else "", + ) +) diff --git a/tests/data/cases/preview_context_managers_38.py b/tests/data/cases/context_managers_38.py similarity index 96% rename from tests/data/cases/preview_context_managers_38.py rename to tests/data/cases/context_managers_38.py index 719d94fdcc5..54fb97c708b 100644 --- a/tests/data/cases/preview_context_managers_38.py +++ b/tests/data/cases/context_managers_38.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.8 +# flags: --minimum-version=3.8 with \ make_context_manager1() as cm1, \ make_context_manager2() as cm2, \ diff --git a/tests/data/cases/preview_context_managers_39.py b/tests/data/cases/context_managers_39.py similarity index 98% rename from tests/data/cases/preview_context_managers_39.py rename to tests/data/cases/context_managers_39.py index 589e00ad187..60fd1a56409 100644 --- a/tests/data/cases/preview_context_managers_39.py +++ b/tests/data/cases/context_managers_39.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.9 +# flags: --minimum-version=3.9 with \ make_context_manager1() as cm1, \ make_context_manager2() as cm2, \ diff --git a/tests/data/cases/preview_context_managers_autodetect_310.py b/tests/data/cases/context_managers_autodetect_310.py similarity index 93% rename from tests/data/cases/preview_context_managers_autodetect_310.py rename to tests/data/cases/context_managers_autodetect_310.py index a9e31076f03..80f211032e5 100644 --- a/tests/data/cases/preview_context_managers_autodetect_310.py +++ b/tests/data/cases/context_managers_autodetect_310.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.10 +# flags: --minimum-version=3.10 # This file uses pattern matching introduced in Python 3.10. diff --git a/tests/data/cases/preview_context_managers_autodetect_311.py b/tests/data/cases/context_managers_autodetect_311.py similarity index 92% rename from tests/data/cases/preview_context_managers_autodetect_311.py rename to tests/data/cases/context_managers_autodetect_311.py index af1e83fe74c..020c4cea967 100644 --- a/tests/data/cases/preview_context_managers_autodetect_311.py +++ b/tests/data/cases/context_managers_autodetect_311.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.11 +# flags: --minimum-version=3.11 # This file uses except* clause in Python 3.11. diff --git a/tests/data/cases/preview_context_managers_autodetect_38.py b/tests/data/cases/context_managers_autodetect_38.py similarity index 98% rename from tests/data/cases/preview_context_managers_autodetect_38.py rename to tests/data/cases/context_managers_autodetect_38.py index 25217a40604..79e438b995e 100644 --- a/tests/data/cases/preview_context_managers_autodetect_38.py +++ b/tests/data/cases/context_managers_autodetect_38.py @@ -1,4 +1,3 @@ -# flags: --preview # This file doesn't use any Python 3.9+ only grammars. diff --git a/tests/data/cases/preview_context_managers_autodetect_39.py b/tests/data/cases/context_managers_autodetect_39.py similarity index 93% rename from tests/data/cases/preview_context_managers_autodetect_39.py rename to tests/data/cases/context_managers_autodetect_39.py index 3f72e48db9d..98e674b2f9d 100644 --- a/tests/data/cases/preview_context_managers_autodetect_39.py +++ b/tests/data/cases/context_managers_autodetect_39.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.9 +# flags: --minimum-version=3.9 # This file uses parenthesized context managers introduced in Python 3.9. diff --git a/tests/data/cases/preview_dummy_implementations.py b/tests/data/cases/dummy_implementations.py similarity index 99% rename from tests/data/cases/preview_dummy_implementations.py rename to tests/data/cases/dummy_implementations.py index 3cd392c9587..0a52c081bcc 100644 --- a/tests/data/cases/preview_dummy_implementations.py +++ b/tests/data/cases/dummy_implementations.py @@ -1,4 +1,3 @@ -# flags: --preview from typing import NoReturn, Protocol, Union, overload class Empty: diff --git a/tests/data/cases/empty_lines.py b/tests/data/cases/empty_lines.py index 4fd47b93dca..4c03e432383 100644 --- a/tests/data/cases/empty_lines.py +++ b/tests/data/cases/empty_lines.py @@ -119,6 +119,7 @@ def f(): if not prev: prevp = preceding_leaf(p) if not prevp or prevp.type in OPENING_BRACKETS: + return NO if prevp.type == token.EQUAL: diff --git a/tests/data/cases/fmtonoff.py b/tests/data/cases/fmtonoff.py index d1f15cd5c8b..8af94563af8 100644 --- a/tests/data/cases/fmtonoff.py +++ b/tests/data/cases/fmtonoff.py @@ -243,12 +243,8 @@ def spaces_types( g: int = 1 if False else 2, h: str = "", i: str = r"", -): - ... - - -def spaces2(result=_core.Value(None)): - ... +): ... +def spaces2(result=_core.Value(None)): ... something = { diff --git a/tests/data/cases/fmtonoff5.py b/tests/data/cases/fmtonoff5.py index 181151b6bd6..4c134a9eea3 100644 --- a/tests/data/cases/fmtonoff5.py +++ b/tests/data/cases/fmtonoff5.py @@ -161,8 +161,7 @@ def this_wont_be_formatted ( self ) -> str: ... class Factory(t.Protocol): - def this_will_be_formatted(self, **kwargs) -> Named: - ... + def this_will_be_formatted(self, **kwargs) -> Named: ... # fmt: on diff --git a/tests/data/cases/preview_form_feeds.py b/tests/data/cases/form_feeds.py similarity index 99% rename from tests/data/cases/preview_form_feeds.py rename to tests/data/cases/form_feeds.py index dc3bd6cfe2e..48ffc98106b 100644 --- a/tests/data/cases/preview_form_feeds.py +++ b/tests/data/cases/form_feeds.py @@ -1,4 +1,3 @@ -# flags: --preview # Warning! This file contains form feeds (ASCII 0x0C, often represented by \f or ^L). diff --git a/tests/data/cases/function.py b/tests/data/cases/function.py index 2d642c8731b..4e3f91fd8b1 100644 --- a/tests/data/cases/function.py +++ b/tests/data/cases/function.py @@ -158,10 +158,7 @@ def spaces_types( g: int = 1 if False else 2, h: str = "", i: str = r"", -): - ... - - +): ... def spaces2(result=_core.Value(None)): assert fut is self._read_fut, (fut, self._read_fut) diff --git a/tests/data/cases/remove_newline_after_match.py b/tests/data/cases/keep_newline_after_match.py similarity index 98% rename from tests/data/cases/remove_newline_after_match.py rename to tests/data/cases/keep_newline_after_match.py index fe6592b664d..dbeccce6264 100644 --- a/tests/data/cases/remove_newline_after_match.py +++ b/tests/data/cases/keep_newline_after_match.py @@ -21,15 +21,21 @@ def http_status(status): # output def http_status(status): + match status: + case 400: + return "Bad request" case 401: + return "Unauthorized" case 403: + return "Forbidden" case 404: + return "Not found" \ No newline at end of file diff --git a/tests/data/cases/long_strings_flag_disabled.py b/tests/data/cases/long_strings_flag_disabled.py index db3954e3abd..d81c331cab2 100644 --- a/tests/data/cases/long_strings_flag_disabled.py +++ b/tests/data/cases/long_strings_flag_disabled.py @@ -43,8 +43,10 @@ % ( "formatted", "string", - ): "This is a really really really long string that has to go inside of a dictionary. It is %s bad (#%d)." - % ("soooo", 2), + ): ( + "This is a really really really long string that has to go inside of a dictionary. It is %s bad (#%d)." + % ("soooo", 2) + ), } func_with_keywords( @@ -254,10 +256,12 @@ + CONCATENATED + "using the '+' operator." ) -annotated_variable: Final = "This is a large string that has a type annotation attached to it. A type annotation should NOT stop a long string from being wrapped." -annotated_variable: Literal[ - "fakse_literal" -] = "This is a large string that has a type annotation attached to it. A type annotation should NOT stop a long string from being wrapped." +annotated_variable: Final = ( + "This is a large string that has a type annotation attached to it. A type annotation should NOT stop a long string from being wrapped." +) +annotated_variable: Literal["fakse_literal"] = ( + "This is a large string that has a type annotation attached to it. A type annotation should NOT stop a long string from being wrapped." +) backslashes = "This is a really long string with \"embedded\" double quotes and 'single' quotes that also handles checking for an even number of backslashes \\" backslashes = "This is a really long string with \"embedded\" double quotes and 'single' quotes that also handles checking for an even number of backslashes \\\\" diff --git a/tests/data/cases/module_docstring_1.py b/tests/data/cases/module_docstring_1.py index d5897b4db60..5751154f7f0 100644 --- a/tests/data/cases/module_docstring_1.py +++ b/tests/data/cases/module_docstring_1.py @@ -1,4 +1,3 @@ -# flags: --preview """Single line module-level docstring should be followed by single newline.""" diff --git a/tests/data/cases/module_docstring_2.py b/tests/data/cases/module_docstring_2.py index 1cc9aea9aea..ac486096c02 100644 --- a/tests/data/cases/module_docstring_2.py +++ b/tests/data/cases/module_docstring_2.py @@ -1,7 +1,7 @@ # flags: --preview """I am a very helpful module docstring. -With trailing spaces: +With trailing spaces (only removed with unify_docstring_detection on): Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, @@ -39,7 +39,7 @@ # output """I am a very helpful module docstring. -With trailing spaces: +With trailing spaces (only removed with unify_docstring_detection on): Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, diff --git a/tests/data/cases/module_docstring_3.py b/tests/data/cases/module_docstring_3.py index 0631e136a3d..3d0058dd554 100644 --- a/tests/data/cases/module_docstring_3.py +++ b/tests/data/cases/module_docstring_3.py @@ -1,4 +1,3 @@ -# flags: --preview """Single line module-level docstring should be followed by single newline.""" a = 1 diff --git a/tests/data/cases/module_docstring_4.py b/tests/data/cases/module_docstring_4.py index 515174dcc04..b1720078f71 100644 --- a/tests/data/cases/module_docstring_4.py +++ b/tests/data/cases/module_docstring_4.py @@ -1,4 +1,3 @@ -# flags: --preview """Single line module-level docstring should be followed by single newline.""" a = 1 diff --git a/tests/data/cases/module_docstring_followed_by_class.py b/tests/data/cases/module_docstring_followed_by_class.py index 6fdbfc8c240..c291e61b960 100644 --- a/tests/data/cases/module_docstring_followed_by_class.py +++ b/tests/data/cases/module_docstring_followed_by_class.py @@ -1,4 +1,3 @@ -# flags: --preview """Two blank lines between module docstring and a class.""" class MyClass: pass diff --git a/tests/data/cases/module_docstring_followed_by_function.py b/tests/data/cases/module_docstring_followed_by_function.py index 5913a59e1fe..fd29b98da8e 100644 --- a/tests/data/cases/module_docstring_followed_by_function.py +++ b/tests/data/cases/module_docstring_followed_by_function.py @@ -1,4 +1,3 @@ -# flags: --preview """Two blank lines between module docstring and a function def.""" def function(): pass diff --git a/tests/data/cases/nested_stub.py b/tests/data/cases/nested_stub.py index ef13c588ce6..40ca11e9330 100644 --- a/tests/data/cases/nested_stub.py +++ b/tests/data/cases/nested_stub.py @@ -1,4 +1,4 @@ -# flags: --pyi --preview +# flags: --pyi import sys class Outer: diff --git a/tests/data/cases/preview_no_blank_line_before_docstring.py b/tests/data/cases/no_blank_line_before_docstring.py similarity index 98% rename from tests/data/cases/preview_no_blank_line_before_docstring.py rename to tests/data/cases/no_blank_line_before_docstring.py index faeaa1e46e4..ced125fef78 100644 --- a/tests/data/cases/preview_no_blank_line_before_docstring.py +++ b/tests/data/cases/no_blank_line_before_docstring.py @@ -1,4 +1,3 @@ -# flags: --preview def line_before_docstring(): """Please move me up""" @@ -63,4 +62,5 @@ class MultilineDocstringsAsWell: class SingleQuotedDocstring: + "I'm a docstring but I don't even get triple quotes." diff --git a/tests/data/cases/preview_pattern_matching_long.py b/tests/data/cases/pattern_matching_long.py similarity index 94% rename from tests/data/cases/preview_pattern_matching_long.py rename to tests/data/cases/pattern_matching_long.py index df849fdc4f2..9a944c9d0c9 100644 --- a/tests/data/cases/preview_pattern_matching_long.py +++ b/tests/data/cases/pattern_matching_long.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.10 +# flags: --minimum-version=3.10 match x: case "abcd" | "abcd" | "abcd" : pass diff --git a/tests/data/cases/preview_pattern_matching_trailing_comma.py b/tests/data/cases/pattern_matching_trailing_comma.py similarity index 92% rename from tests/data/cases/preview_pattern_matching_trailing_comma.py rename to tests/data/cases/pattern_matching_trailing_comma.py index e6c0d88bb80..5660b0f6a14 100644 --- a/tests/data/cases/preview_pattern_matching_trailing_comma.py +++ b/tests/data/cases/pattern_matching_trailing_comma.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.10 +# flags: --minimum-version=3.10 match maybe, multiple: case perhaps, 5: pass diff --git a/tests/data/cases/pep604_union_types_line_breaks.py b/tests/data/cases/pep604_union_types_line_breaks.py index fee2b840494..745bc9e8b02 100644 --- a/tests/data/cases/pep604_union_types_line_breaks.py +++ b/tests/data/cases/pep604_union_types_line_breaks.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.10 +# flags: --minimum-version=3.10 # This has always worked z= Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong diff --git a/tests/data/cases/pep_572_py310.py b/tests/data/cases/pep_572_py310.py index 9f999deeb89..ba488d4741c 100644 --- a/tests/data/cases/pep_572_py310.py +++ b/tests/data/cases/pep_572_py310.py @@ -1,8 +1,8 @@ # flags: --minimum-version=3.10 # Unparenthesized walruses are now allowed in indices since Python 3.10. -x[a:=0] -x[a:=0, b:=1] -x[5, b:=0] +x[a := 0] +x[a := 0, b := 1] +x[5, b := 0] # Walruses are allowed inside generator expressions on function calls since 3.10. if any(match := pattern_error.match(s) for s in buffer): diff --git a/tests/data/cases/pep_572_remove_parens.py b/tests/data/cases/pep_572_remove_parens.py index 24f1ac29168..f0026ceb032 100644 --- a/tests/data/cases/pep_572_remove_parens.py +++ b/tests/data/cases/pep_572_remove_parens.py @@ -96,18 +96,16 @@ async def await_the_walrus(): foo(x=(y := f(x))) -def foo(answer=(p := 42)): - ... +def foo(answer=(p := 42)): ... -def foo2(answer: (p := 42) = 5): - ... +def foo2(answer: (p := 42) = 5): ... lambda: (x := 1) a[(x := 12)] -a[:(x := 13)] +a[: (x := 13)] # we don't touch expressions in f-strings but if we do one day, don't break 'em f"{(x:=10)}" diff --git a/tests/data/cases/preview_pep_572.py b/tests/data/cases/pep_572_slices.py similarity index 75% rename from tests/data/cases/preview_pep_572.py rename to tests/data/cases/pep_572_slices.py index 75ad0cc4176..aa772b1f1f5 100644 --- a/tests/data/cases/preview_pep_572.py +++ b/tests/data/cases/pep_572_slices.py @@ -1,4 +1,3 @@ -# flags: --preview x[(a:=0):] x[:(a:=0)] diff --git a/tests/data/cases/preview_percent_precedence.py b/tests/data/cases/percent_precedence.py similarity index 91% rename from tests/data/cases/preview_percent_precedence.py rename to tests/data/cases/percent_precedence.py index aeaf450ff5e..7822e42c69d 100644 --- a/tests/data/cases/preview_percent_precedence.py +++ b/tests/data/cases/percent_precedence.py @@ -1,4 +1,3 @@ -# flags: --preview ("" % a) ** 2 ("" % a)[0] ("" % a)() @@ -31,9 +30,9 @@ 2 // ("" % a) 2 % ("" % a) +("" % a) -b + "" % a +b + ("" % a) -("" % a) -b - "" % a +b - ("" % a) b + -("" % a) ~("" % a) 2 ** ("" % a) diff --git a/tests/data/cases/preview_power_op_spacing.py b/tests/data/cases/power_op_spacing_long.py similarity index 99% rename from tests/data/cases/preview_power_op_spacing.py rename to tests/data/cases/power_op_spacing_long.py index 650c6fecb20..30e6eb788b3 100644 --- a/tests/data/cases/preview_power_op_spacing.py +++ b/tests/data/cases/power_op_spacing_long.py @@ -1,4 +1,3 @@ -# flags: --preview a = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 b = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 c = 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 diff --git a/tests/data/cases/preview_prefer_rhs_split.py b/tests/data/cases/prefer_rhs_split.py similarity index 99% rename from tests/data/cases/preview_prefer_rhs_split.py rename to tests/data/cases/prefer_rhs_split.py index 28d89c368c0..f3d9fd67251 100644 --- a/tests/data/cases/preview_prefer_rhs_split.py +++ b/tests/data/cases/prefer_rhs_split.py @@ -1,4 +1,3 @@ -# flags: --preview first_item, second_item = ( some_looooooooong_module.some_looooooooooooooong_function_name( first_argument, second_argument, third_argument diff --git a/tests/data/cases/py310_pep572.py b/tests/data/cases/py310_pep572.py index 172be3898d6..73fbe44d42c 100644 --- a/tests/data/cases/py310_pep572.py +++ b/tests/data/cases/py310_pep572.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.10 +# flags: --minimum-version=3.10 x[a:=0] x[a := 0] x[a := 0, b := 1] diff --git a/tests/data/cases/python39.py b/tests/data/cases/python39.py index 1b9536c1529..85eddc38e00 100644 --- a/tests/data/cases/python39.py +++ b/tests/data/cases/python39.py @@ -15,19 +15,16 @@ def f(): # output @relaxed_decorator[0] -def f(): - ... +def f(): ... @relaxed_decorator[ extremely_long_name_that_definitely_will_not_fit_on_one_line_of_standard_length ] -def f(): - ... +def f(): ... @extremely_long_variable_name_that_doesnt_fit := complex.expression( with_long="arguments_value_that_wont_fit_at_the_end_of_the_line" ) -def f(): - ... \ No newline at end of file +def f(): ... \ No newline at end of file diff --git a/tests/data/cases/raw_docstring.py b/tests/data/cases/raw_docstring.py index 751fd3201df..7f88bb2de86 100644 --- a/tests/data/cases/raw_docstring.py +++ b/tests/data/cases/raw_docstring.py @@ -1,4 +1,4 @@ -# flags: --preview --skip-string-normalization +# flags: --skip-string-normalization class C: r"""Raw""" diff --git a/tests/data/cases/preview_docstring_no_string_normalization.py b/tests/data/cases/raw_docstring_no_string_normalization.py similarity index 88% rename from tests/data/cases/preview_docstring_no_string_normalization.py rename to tests/data/cases/raw_docstring_no_string_normalization.py index 712c7364f51..a201c1e8fae 100644 --- a/tests/data/cases/preview_docstring_no_string_normalization.py +++ b/tests/data/cases/raw_docstring_no_string_normalization.py @@ -1,4 +1,4 @@ -# flags: --preview --skip-string-normalization +# flags: --skip-string-normalization def do_not_touch_this_prefix(): R"""There was a bug where docstring prefixes would be normalized even with -S.""" diff --git a/tests/data/cases/remove_newline_after_code_block_open.py b/tests/data/cases/remove_newline_after_code_block_open.py index ef2e5c2f6f5..6622e8afb7d 100644 --- a/tests/data/cases/remove_newline_after_code_block_open.py +++ b/tests/data/cases/remove_newline_after_code_block_open.py @@ -3,14 +3,14 @@ def foo1(): - print("The newline above me should be deleted!") + print("The newline above me should be kept!") def foo2(): - print("All the newlines above me should be deleted!") + print("All the newlines above me should be kept!") def foo3(): @@ -30,31 +30,31 @@ def foo4(): class Foo: def bar(self): - print("The newline above me should be deleted!") + print("The newline above me should be kept!") for i in range(5): - print(f"{i}) The line above me should be removed!") + print(f"{i}) The line above me should be kept!") for i in range(5): - print(f"{i}) The lines above me should be removed!") + print(f"{i}) The lines above me should be kept!") for i in range(5): for j in range(7): - print(f"{i}) The lines above me should be removed!") + print(f"{i}) The lines above me should be kept!") if random.randint(0, 3) == 0: - print("The new line above me is about to be removed!") + print("The new line above me will be kept!") if random.randint(0, 3) == 0: @@ -62,43 +62,45 @@ def bar(self): - print("The new lines above me is about to be removed!") + print("The new lines above me will be kept!") if random.randint(0, 3) == 0: + if random.uniform(0, 1) > 0.5: - print("Two lines above me are about to be removed!") + + print("Two lines above me will be kept!") while True: - print("The newline above me should be deleted!") + print("The newline above me should be kept!") while True: - print("The newlines above me should be deleted!") + print("The newlines above me should be kept!") while True: while False: - print("The newlines above me should be deleted!") + print("The newlines above me should be kept!") with open("/path/to/file.txt", mode="w") as file: - file.write("The new line above me is about to be removed!") + file.write("The new line above me will be kept!") with open("/path/to/file.txt", mode="w") as file: - file.write("The new lines above me is about to be removed!") + file.write("The new lines above me will be kept!") with open("/path/to/file.txt", mode="r") as read_file: @@ -113,20 +115,24 @@ def bar(self): def foo1(): - print("The newline above me should be deleted!") + + print("The newline above me should be kept!") def foo2(): - print("All the newlines above me should be deleted!") + + print("All the newlines above me should be kept!") def foo3(): + print("No newline above me!") print("There is a newline above me, and that's OK!") def foo4(): + # There is a comment here print("The newline above me should not be deleted!") @@ -134,56 +140,73 @@ def foo4(): class Foo: def bar(self): - print("The newline above me should be deleted!") + + print("The newline above me should be kept!") for i in range(5): - print(f"{i}) The line above me should be removed!") + + print(f"{i}) The line above me should be kept!") for i in range(5): - print(f"{i}) The lines above me should be removed!") + + print(f"{i}) The lines above me should be kept!") for i in range(5): + for j in range(7): - print(f"{i}) The lines above me should be removed!") + + print(f"{i}) The lines above me should be kept!") if random.randint(0, 3) == 0: - print("The new line above me is about to be removed!") + + print("The new line above me will be kept!") if random.randint(0, 3) == 0: - print("The new lines above me is about to be removed!") + + print("The new lines above me will be kept!") if random.randint(0, 3) == 0: + if random.uniform(0, 1) > 0.5: - print("Two lines above me are about to be removed!") + + print("Two lines above me will be kept!") while True: - print("The newline above me should be deleted!") + + print("The newline above me should be kept!") while True: - print("The newlines above me should be deleted!") + + print("The newlines above me should be kept!") while True: + while False: - print("The newlines above me should be deleted!") + + print("The newlines above me should be kept!") with open("/path/to/file.txt", mode="w") as file: - file.write("The new line above me is about to be removed!") + + file.write("The new line above me will be kept!") with open("/path/to/file.txt", mode="w") as file: - file.write("The new lines above me is about to be removed!") + + file.write("The new lines above me will be kept!") with open("/path/to/file.txt", mode="r") as read_file: + with open("/path/to/output_file.txt", mode="w") as write_file: + write_file.writelines(read_file.readlines()) diff --git a/tests/data/cases/return_annotation_brackets.py b/tests/data/cases/return_annotation_brackets.py index 8509ecdb92c..ed05bed61f4 100644 --- a/tests/data/cases/return_annotation_brackets.py +++ b/tests/data/cases/return_annotation_brackets.py @@ -88,7 +88,6 @@ def foo() -> tuple[int, int, int,]: return 2 # Magic trailing comma example, with params -# this is broken - the trailing comma is transferred to the param list. Fixed in preview def foo(a,b) -> tuple[int, int, int,]: return 2 @@ -194,30 +193,27 @@ def foo() -> tuple[int, int, int]: return 2 -def foo() -> ( - tuple[ - loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong, - loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong, - loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong, - ] -): +def foo() -> tuple[ + loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong, + loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong, + loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong, +]: return 2 # Magic trailing comma example -def foo() -> ( - tuple[ - int, - int, - int, - ] -): +def foo() -> tuple[ + int, + int, + int, +]: return 2 # Magic trailing comma example, with params -# this is broken - the trailing comma is transferred to the param list. Fixed in preview -def foo( - a, b -) -> tuple[int, int, int,]: +def foo(a, b) -> tuple[ + int, + int, + int, +]: return 2 diff --git a/tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py b/tests/data/cases/single_line_format_skip_with_multiple_comments.py similarity index 97% rename from tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py rename to tests/data/cases/single_line_format_skip_with_multiple_comments.py index efde662baa8..7212740fc42 100644 --- a/tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py +++ b/tests/data/cases/single_line_format_skip_with_multiple_comments.py @@ -1,4 +1,3 @@ -# flags: --preview foo = 123 # fmt: skip # noqa: E501 # pylint bar = ( 123 , diff --git a/tests/data/cases/preview_trailing_comma.py b/tests/data/cases/trailing_comma.py similarity index 97% rename from tests/data/cases/preview_trailing_comma.py rename to tests/data/cases/trailing_comma.py index bba7e7ad16d..5b09c664606 100644 --- a/tests/data/cases/preview_trailing_comma.py +++ b/tests/data/cases/trailing_comma.py @@ -1,4 +1,3 @@ -# flags: --preview e = { "a": fun(msg, "ts"), "longggggggggggggggid": ..., diff --git a/tests/data/cases/walrus_in_dict.py b/tests/data/cases/walrus_in_dict.py index c33eecd84a6..c91ad9e8611 100644 --- a/tests/data/cases/walrus_in_dict.py +++ b/tests/data/cases/walrus_in_dict.py @@ -1,7 +1,9 @@ # flags: --preview +# This is testing an issue that is specific to the preview style { "is_update": (up := commit.hash in update_hashes) } # output +# This is testing an issue that is specific to the preview style {"is_update": (up := commit.hash in update_hashes)} From a5196e6f1f450e4c8da0e514e01873a0cc1e1a3c Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Thu, 25 Jan 2024 03:31:49 -0600 Subject: [PATCH 21/68] fix: Don't normalize whitespace before fmt:skip comments (#4146) Signed-off-by: RedGuy12 --- CHANGES.md | 1 + src/black/comments.py | 14 +++++++++++--- src/black/mode.py | 1 + tests/data/cases/fmtskip9.py | 9 +++++++++ 4 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 tests/data/cases/fmtskip9.py diff --git a/CHANGES.md b/CHANGES.md index 0e2974d706e..2fe14cd5246 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -56,6 +56,7 @@ release: - Format module docstrings the same as class and function docstrings (#4095) - Fix crash when using a walrus in a dictionary (#4155) - Fix unnecessary parentheses when wrapping long dicts (#4135) +- Stop normalizing spaces before `# fmt: skip` comments (#4146) ### Configuration diff --git a/src/black/comments.py b/src/black/comments.py index 910e1b760f0..ea54e2468c9 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -3,7 +3,7 @@ from functools import lru_cache from typing import Collection, Final, Iterator, List, Optional, Tuple, Union -from black.mode import Mode +from black.mode import Mode, Preview from black.nodes import ( CLOSING_BRACKETS, STANDALONE_COMMENT, @@ -46,6 +46,7 @@ class ProtoComment: newlines: int # how many newlines before the comment consumed: int # how many characters of the original leaf's prefix did we consume form_feed: bool # is there a form feed before the comment + leading_whitespace: str # leading whitespace before the comment, if any def generate_comments(leaf: LN) -> Iterator[Leaf]: @@ -88,7 +89,9 @@ def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]: form_feed = False for index, full_line in enumerate(re.split("\r?\n", prefix)): consumed += len(full_line) + 1 # adding the length of the split '\n' - line = full_line.lstrip() + match = re.match(r"^(\s*)(\S.*|)$", full_line) + assert match + whitespace, line = match.groups() if not line: nlines += 1 if "\f" in full_line: @@ -113,6 +116,7 @@ def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]: newlines=nlines, consumed=consumed, form_feed=form_feed, + leading_whitespace=whitespace, ) ) form_feed = False @@ -230,7 +234,11 @@ def convert_one_fmt_off_pair( standalone_comment_prefix += fmt_off_prefix hidden_value = comment.value + "\n" + hidden_value if _contains_fmt_skip_comment(comment.value, mode): - hidden_value += " " + comment.value + hidden_value += ( + comment.leading_whitespace + if Preview.no_normalize_fmt_skip_whitespace in mode + else " " + ) + comment.value if hidden_value.endswith("\n"): # That happens when one of the `ignored_nodes` ended with a NEWLINE # leaf (possibly followed by a DEDENT). diff --git a/src/black/mode.py b/src/black/mode.py index 1b97f3508ee..a1519f17bcc 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -174,6 +174,7 @@ class Preview(Enum): string_processing = auto() hug_parens_with_braces_and_square_brackets = auto() unify_docstring_detection = auto() + no_normalize_fmt_skip_whitespace = auto() wrap_long_dict_values_in_parens = auto() multiline_string_handling = auto() diff --git a/tests/data/cases/fmtskip9.py b/tests/data/cases/fmtskip9.py new file mode 100644 index 00000000000..30085bdd973 --- /dev/null +++ b/tests/data/cases/fmtskip9.py @@ -0,0 +1,9 @@ +# flags: --preview +print () # fmt: skip +print () # fmt:skip + + +# output + +print () # fmt: skip +print () # fmt:skip From f7d552d9b7bd33f43edd0867757c27b1aa36c651 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 25 Jan 2024 13:11:26 -0800 Subject: [PATCH 22/68] Remove reference (#4169) This is out-of-date and just a chore. I don't think this is useful to contributors and Black doesn't even have a public API. --- docs/contributing/index.md | 5 - .../reference/reference_classes.rst | 234 ------------------ .../reference/reference_exceptions.rst | 18 -- .../reference/reference_functions.rst | 172 ------------- .../reference/reference_summary.rst | 19 -- 5 files changed, 448 deletions(-) delete mode 100644 docs/contributing/reference/reference_classes.rst delete mode 100644 docs/contributing/reference/reference_exceptions.rst delete mode 100644 docs/contributing/reference/reference_functions.rst delete mode 100644 docs/contributing/reference/reference_summary.rst diff --git a/docs/contributing/index.md b/docs/contributing/index.md index 3314c8eaa39..f15afa318d4 100644 --- a/docs/contributing/index.md +++ b/docs/contributing/index.md @@ -9,7 +9,6 @@ the_basics gauging_changes issue_triage release_process -reference/reference_summary ``` Welcome! Happy to see you willing to make the project better. Have you read the entire @@ -42,9 +41,5 @@ This section covers the following topics: - {doc}`the_basics` - {doc}`gauging_changes` - {doc}`release_process` -- {doc}`reference/reference_summary` For an overview on contributing to the _Black_, please checkout {doc}`the_basics`. - -If you need a reference of the functions, classes, etc. available to you while -developing _Black_, there's the {doc}`reference/reference_summary` docs. diff --git a/docs/contributing/reference/reference_classes.rst b/docs/contributing/reference/reference_classes.rst deleted file mode 100644 index dc615579e30..00000000000 --- a/docs/contributing/reference/reference_classes.rst +++ /dev/null @@ -1,234 +0,0 @@ -*Black* classes -=============== - -*Contents are subject to change.* - -Black Classes -~~~~~~~~~~~~~~ - -.. currentmodule:: black - -:class:`BracketTracker` ------------------------ - -.. autoclass:: black.brackets.BracketTracker - :members: - -:class:`Line` -------------- - -.. autoclass:: black.lines.Line - :members: - :special-members: __str__, __bool__ - -:class:`RHSResult` -------------------------- - -.. autoclass:: black.lines.RHSResult - :members: - -:class:`LinesBlock` -------------------------- - -.. autoclass:: black.lines.LinesBlock - :members: - -:class:`EmptyLineTracker` -------------------------- - -.. autoclass:: black.lines.EmptyLineTracker - :members: - -:class:`LineGenerator` ----------------------- - -.. autoclass:: black.linegen.LineGenerator - :show-inheritance: - :members: - -:class:`ProtoComment` ---------------------- - -.. autoclass:: black.comments.ProtoComment - :members: - -:class:`Mode` ---------------------- - -.. autoclass:: black.mode.Mode - :members: - -:class:`Report` ---------------- - -.. autoclass:: black.report.Report - :members: - :special-members: __str__ - -:class:`Ok` ---------------- - -.. autoclass:: black.rusty.Ok - :show-inheritance: - :members: - -:class:`Err` ---------------- - -.. autoclass:: black.rusty.Err - :show-inheritance: - :members: - -:class:`Visitor` ----------------- - -.. autoclass:: black.nodes.Visitor - :show-inheritance: - :members: - -:class:`StringTransformer` ----------------------------- - -.. autoclass:: black.trans.StringTransformer - :show-inheritance: - :members: - -:class:`CustomSplit` ----------------------------- - -.. autoclass:: black.trans.CustomSplit - :members: - -:class:`CustomSplitMapMixin` ------------------------------ - -.. autoclass:: black.trans.CustomSplitMapMixin - :show-inheritance: - :members: - -:class:`StringMerger` ----------------------- - -.. autoclass:: black.trans.StringMerger - :show-inheritance: - :members: - -:class:`StringParenStripper` ------------------------------ - -.. autoclass:: black.trans.StringParenStripper - :show-inheritance: - :members: - -:class:`BaseStringSplitter` ------------------------------ - -.. autoclass:: black.trans.BaseStringSplitter - :show-inheritance: - :members: - -:class:`StringSplitter` ------------------------------ - -.. autoclass:: black.trans.StringSplitter - :show-inheritance: - :members: - -:class:`StringParenWrapper` ------------------------------ - -.. autoclass:: black.trans.StringParenWrapper - :show-inheritance: - :members: - -:class:`StringParser` ------------------------------ - -.. autoclass:: black.trans.StringParser - :members: - -:class:`DebugVisitor` ------------------------- - -.. autoclass:: black.debug.DebugVisitor - :show-inheritance: - :members: - -:class:`Replacement` ------------------------- - -.. autoclass:: black.handle_ipynb_magics.Replacement - :members: - -:class:`CellMagic` ------------------------- - -.. autoclass:: black.handle_ipynb_magics.CellMagic - :members: - -:class:`CellMagicFinder` ------------------------- - -.. autoclass:: black.handle_ipynb_magics.CellMagicFinder - :show-inheritance: - :members: - -:class:`OffsetAndMagic` ------------------------- - -.. autoclass:: black.handle_ipynb_magics.OffsetAndMagic - :members: - -:class:`MagicFinder` ------------------------- - -.. autoclass:: black.handle_ipynb_magics.MagicFinder - :show-inheritance: - :members: - -:class:`Cache` ------------------------- - -.. autoclass:: black.cache.Cache - :show-inheritance: - :members: - -Enum Classes -~~~~~~~~~~~~~ - -Classes inherited from Python `Enum `_ class. - -:class:`Changed` ----------------- - -.. autoclass:: black.report.Changed - :show-inheritance: - :members: - -:class:`WriteBack` ------------------- - -.. autoclass:: black.WriteBack - :show-inheritance: - :members: - -:class:`TargetVersion` ----------------------- - -.. autoclass:: black.mode.TargetVersion - :show-inheritance: - :members: - -:class:`Feature` ------------------- - -.. autoclass:: black.mode.Feature - :show-inheritance: - :members: - -:class:`Preview` ------------------- - -.. autoclass:: black.mode.Preview - :show-inheritance: - :members: diff --git a/docs/contributing/reference/reference_exceptions.rst b/docs/contributing/reference/reference_exceptions.rst deleted file mode 100644 index ab46ebdb628..00000000000 --- a/docs/contributing/reference/reference_exceptions.rst +++ /dev/null @@ -1,18 +0,0 @@ -*Black* exceptions -================== - -*Contents are subject to change.* - -.. currentmodule:: black - -.. autoexception:: black.trans.CannotTransform - -.. autoexception:: black.linegen.CannotSplit - -.. autoexception:: black.brackets.BracketMatchError - -.. autoexception:: black.report.NothingChanged - -.. autoexception:: black.parsing.InvalidInput - -.. autoexception:: black.mode.Deprecated diff --git a/docs/contributing/reference/reference_functions.rst b/docs/contributing/reference/reference_functions.rst deleted file mode 100644 index ebadf6975a7..00000000000 --- a/docs/contributing/reference/reference_functions.rst +++ /dev/null @@ -1,172 +0,0 @@ -*Black* functions -================= - -*Contents are subject to change.* - -.. currentmodule:: black - -Assertions and checks ---------------------- - -.. autofunction:: black.assert_equivalent - -.. autofunction:: black.assert_stable - -.. autofunction:: black.lines.can_be_split - -.. autofunction:: black.lines.can_omit_invisible_parens - -.. autofunction:: black.nodes.is_empty_tuple - -.. autofunction:: black.nodes.is_import - -.. autofunction:: black.lines.is_line_short_enough - -.. autofunction:: black.nodes.is_multiline_string - -.. autofunction:: black.nodes.is_one_tuple - -.. autofunction:: black.brackets.is_split_after_delimiter - -.. autofunction:: black.brackets.is_split_before_delimiter - -.. autofunction:: black.nodes.is_stub_body - -.. autofunction:: black.nodes.is_stub_suite - -.. autofunction:: black.nodes.is_vararg - -.. autofunction:: black.nodes.is_yield - - -Formatting ----------- - -.. autofunction:: black.format_file_contents - -.. autofunction:: black.format_file_in_place - -.. autofunction:: black.format_stdin_to_stdout - -.. autofunction:: black.format_str - -.. autofunction:: black.reformat_one - -.. autofunction:: black.concurrency.schedule_formatting - -File operations ---------------- - -.. autofunction:: black.dump_to_file - -.. autofunction:: black.find_project_root - -.. autofunction:: black.gen_python_files - -.. autofunction:: black.read_pyproject_toml - -Parsing -------- - -.. autofunction:: black.decode_bytes - -.. autofunction:: black.parsing.lib2to3_parse - -.. autofunction:: black.parsing.lib2to3_unparse - -Split functions ---------------- - -.. autofunction:: black.linegen.bracket_split_build_line - -.. autofunction:: black.linegen.bracket_split_succeeded_or_raise - -.. autofunction:: black.linegen.delimiter_split - -.. autofunction:: black.linegen.left_hand_split - -.. autofunction:: black.linegen.right_hand_split - -.. autofunction:: black.linegen.standalone_comment_split - -.. autofunction:: black.linegen.transform_line - -Caching -------- - -.. autofunction:: black.cache.get_cache_dir - -.. autofunction:: black.cache.get_cache_file - -Utilities ---------- - -.. py:function:: black.debug.DebugVisitor.show(code: str) -> None - - Pretty-print the lib2to3 AST of a given string of `code`. - -.. autofunction:: black.concurrency.cancel - -.. autofunction:: black.nodes.child_towards - -.. autofunction:: black.nodes.container_of - -.. autofunction:: black.comments.convert_one_fmt_off_pair - -.. autofunction:: black.diff - -.. autofunction:: black.linegen.dont_increase_indentation - -.. autofunction:: black.numerics.format_float_or_int_string - -.. autofunction:: black.nodes.ensure_visible - -.. autofunction:: black.lines.enumerate_reversed - -.. autofunction:: black.comments.generate_comments - -.. autofunction:: black.comments.generate_ignored_nodes - -.. autofunction:: black.comments.is_fmt_on - -.. autofunction:: black.comments.children_contains_fmt_on - -.. autofunction:: black.nodes.first_leaf_of - -.. autofunction:: black.linegen.generate_trailers_to_omit - -.. autofunction:: black.get_future_imports - -.. autofunction:: black.comments.list_comments - -.. autofunction:: black.comments.make_comment - -.. autofunction:: black.linegen.maybe_make_parens_invisible_in_atom - -.. autofunction:: black.brackets.max_delimiter_priority_in_atom - -.. autofunction:: black.normalize_fmt_off - -.. autofunction:: black.numerics.normalize_numeric_literal - -.. autofunction:: black.comments.normalize_trailing_prefix - -.. autofunction:: black.strings.normalize_string_prefix - -.. autofunction:: black.strings.normalize_string_quotes - -.. autofunction:: black.linegen.normalize_invisible_parens - -.. autofunction:: black.nodes.preceding_leaf - -.. autofunction:: black.re_compile_maybe_verbose - -.. autofunction:: black.linegen.should_split_line - -.. autofunction:: black.concurrency.shutdown - -.. autofunction:: black.strings.sub_twice - -.. autofunction:: black.nodes.whitespace - -.. autofunction:: black.nodes.make_simple_prefix diff --git a/docs/contributing/reference/reference_summary.rst b/docs/contributing/reference/reference_summary.rst deleted file mode 100644 index c6163d897b6..00000000000 --- a/docs/contributing/reference/reference_summary.rst +++ /dev/null @@ -1,19 +0,0 @@ -Developer reference -=================== - -.. note:: - - As of June 2023, the documentation of *Black classes* and *Black exceptions* - has been updated to the latest available version. - - The documentation of *Black functions* is quite outdated and has been neglected. Many - functions worthy of inclusion aren't documented. Contributions are appreciated! - -*Contents are subject to change.* - -.. toctree:: - :maxdepth: 2 - - reference_classes - reference_functions - reference_exceptions From 17f7f297efd29a9b4187af8420e88ca156c1d221 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 25 Jan 2024 13:41:45 -0800 Subject: [PATCH 23/68] Simplify code in lines.py (#4167) This has been getting a little messy. These changes neaten things up, we don't have to keep guarding against `self.previous_line is not None`, we make it clearer what logic has side effects, we reduce the amount of code that tricky `before` could touch, etc --- src/black/lines.py | 62 +++++++++++++++------------------------------- 1 file changed, 20 insertions(+), 42 deletions(-) diff --git a/src/black/lines.py b/src/black/lines.py index 29f87137614..72634fd36d2 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -565,14 +565,9 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: ) before, after = self._maybe_empty_lines(current_line) previous_after = self.previous_block.after if self.previous_block else 0 - before = ( - # Black should not insert empty lines at the beginning - # of the file - 0 - if self.previous_line is None - else before - previous_after - ) + before = max(0, before - previous_after) if ( + # Always have one empty line after a module docstring self.previous_block and self.previous_block.previous_block is None and len(self.previous_block.original_line.leaves) == 1 @@ -607,10 +602,11 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: self.previous_block = block return block - def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: + def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: # noqa: C901 max_allowed = 1 if current_line.depth == 0: max_allowed = 1 if self.mode.is_pyi else 2 + if current_line.leaves: # Consume the first leaf's extra newlines. first_leaf = current_line.leaves[0] @@ -623,9 +619,22 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: user_had_newline = bool(before) depth = current_line.depth + # Mutate self.previous_defs, remainder of this function should be pure previous_def = None while self.previous_defs and self.previous_defs[-1].depth >= depth: previous_def = self.previous_defs.pop() + if current_line.is_def or current_line.is_class: + self.previous_defs.append(current_line) + + if self.previous_line is None: + # Don't insert empty lines before the first line in the file. + return 0, 0 + + if current_line.is_docstring: + if self.previous_line.is_class: + return 0, 1 + if self.previous_line.opens_block and self.previous_line.is_def: + return 0, 0 if previous_def is not None: assert self.previous_line is not None @@ -668,49 +677,24 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: ) if ( - self.previous_line - and self.previous_line.is_import + self.previous_line.is_import and not current_line.is_import and not current_line.is_fmt_pass_converted(first_leaf_matches=is_import) and depth == self.previous_line.depth ): return (before or 1), 0 - if ( - self.previous_line - and self.previous_line.is_class - and current_line.is_docstring - ): - return 0, 1 - - # In preview mode, always allow blank lines, except right before a function - # docstring - is_empty_first_line_ok = not current_line.is_docstring or ( - self.previous_line and not self.previous_line.is_def - ) - - if ( - self.previous_line - and self.previous_line.opens_block - and not is_empty_first_line_ok - ): - return 0, 0 return before, 0 def _maybe_empty_lines_for_class_or_def( # noqa: C901 self, current_line: Line, before: int, user_had_newline: bool ) -> Tuple[int, int]: - if not current_line.is_decorator: - self.previous_defs.append(current_line) - if self.previous_line is None: - # Don't insert empty lines before the first line in the file. - return 0, 0 + assert self.previous_line is not None if self.previous_line.is_decorator: if self.mode.is_pyi and current_line.is_stub_class: # Insert an empty line after a decorated stub class return 0, 1 - return 0, 0 if self.previous_line.depth < current_line.depth and ( @@ -718,8 +702,7 @@ def _maybe_empty_lines_for_class_or_def( # noqa: C901 ): if self.mode.is_pyi: return 0, 0 - else: - return 1 if user_had_newline else 0, 0 + return 1 if user_had_newline else 0, 0 comment_to_add_newlines: Optional[LinesBlock] = None if ( @@ -750,9 +733,6 @@ def _maybe_empty_lines_for_class_or_def( # noqa: C901 newlines = 0 else: newlines = 1 - # Remove case `self.previous_line.depth > current_line.depth` below when - # this becomes stable. - # # Don't inspect the previous line if it's part of the body of the previous # statement in the same level, we always want a blank line if there's # something with a body preceding. @@ -769,8 +749,6 @@ def _maybe_empty_lines_for_class_or_def( # noqa: C901 # Blank line between a block of functions (maybe with preceding # decorators) and a block of non-functions newlines = 1 - elif self.previous_line.depth > current_line.depth: - newlines = 1 else: newlines = 0 else: From 7d789469ed947022f183962b823f5862511272ac Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 25 Jan 2024 16:15:18 -0800 Subject: [PATCH 24/68] Describe 2024 module docstring more accurately (#4168) --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 2fe14cd5246..95ef0095102 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,7 +17,7 @@ changes: - Hex codes in Unicode escape sequences are now standardized to lowercase (#2916) - Allow empty first lines at the beginning of most blocks (#3967, #4061) - Add parentheses around long type annotations (#3899) -- Standardize on a single newline after module docstrings (#3932) +- Enforce newline after module docstrings (#3932, #4028) - Fix incorrect magic trailing comma handling in return types (#3916) - Remove blank lines before class docstrings (#3692) - Wrap multiple context managers in parentheses if combined in a single `with` statement From bccec8adfbed2bbc24c0859e8758d5e7809d42b7 Mon Sep 17 00:00:00 2001 From: Daniel Krzeminski Date: Thu, 25 Jan 2024 18:41:37 -0600 Subject: [PATCH 25/68] Show warning on invalid toml configuration (#4165) Co-authored-by: Jelle Zijlstra --- CHANGES.md | 1 + src/black/__init__.py | 17 +++++++++++++++++ tests/data/incorrect_spelling.toml | 5 +++++ tests/test_black.py | 17 +++++++++++++++++ 4 files changed, 40 insertions(+) create mode 100644 tests/data/incorrect_spelling.toml diff --git a/CHANGES.md b/CHANGES.md index 95ef0095102..45cb967e74a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -62,6 +62,7 @@ release: +- Print warning when toml config contains an invalid key (#4165) - Fix symlink handling, properly catch and ignore symlinks that point outside of root (#4161) - Fix cache mtime logic that resulted in false positive cache hits (#4128) diff --git a/src/black/__init__.py b/src/black/__init__.py index e3cbaab5f1d..961ed9479a8 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -142,6 +142,7 @@ def read_pyproject_toml( if not config: return None else: + spellcheck_pyproject_toml_keys(ctx, list(config), value) # Sanitize the values to be Click friendly. For more information please see: # https://github.com/psf/black/issues/1458 # https://github.com/pallets/click/issues/1567 @@ -181,6 +182,22 @@ def read_pyproject_toml( return value +def spellcheck_pyproject_toml_keys( + ctx: click.Context, config_keys: List[str], config_file_path: str +) -> None: + invalid_keys: List[str] = [] + available_config_options = {param.name for param in ctx.command.params} + for key in config_keys: + if key not in available_config_options: + invalid_keys.append(key) + if invalid_keys: + keys_str = ", ".join(map(repr, invalid_keys)) + out( + f"Invalid config keys detected: {keys_str} (in {config_file_path})", + fg="red", + ) + + def target_version_option_callback( c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...] ) -> List[TargetVersion]: diff --git a/tests/data/incorrect_spelling.toml b/tests/data/incorrect_spelling.toml new file mode 100644 index 00000000000..560c9e27be2 --- /dev/null +++ b/tests/data/incorrect_spelling.toml @@ -0,0 +1,5 @@ +[tool.black] +ine_length = 50 +target-ersion = ['py37'] +exclude='\.pyi?$' +include='\.py?$' \ No newline at end of file diff --git a/tests/test_black.py b/tests/test_black.py index 2b5fab5d28d..a979a95b674 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -106,6 +106,7 @@ class FakeContext(click.Context): def __init__(self) -> None: self.default_map: Dict[str, Any] = {} self.params: Dict[str, Any] = {} + self.command: click.Command = black.main # Dummy root, since most of the tests don't care about it self.obj: Dict[str, Any] = {"root": PROJECT_ROOT} @@ -1538,6 +1539,22 @@ def test_parse_pyproject_toml(self) -> None: self.assertEqual(config["exclude"], r"\.pyi?$") self.assertEqual(config["include"], r"\.py?$") + def test_spellcheck_pyproject_toml(self) -> None: + test_toml_file = THIS_DIR / "data" / "incorrect_spelling.toml" + result = BlackRunner().invoke( + black.main, + [ + "--code=print('hello world')", + "--verbose", + f"--config={str(test_toml_file)}", + ], + ) + + assert ( + r"Invalid config keys detected: 'ine_length', 'target_ersion' (in" + rf" {test_toml_file})" in result.stderr + ) + def test_parse_pyproject_toml_project_metadata(self) -> None: for test_toml, expected in [ ("only_black_pyproject.toml", ["py310"]), From 4f47cac1925a2232892ceae438e2c62f81517714 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 25 Jan 2024 17:00:47 -0800 Subject: [PATCH 26/68] Add --unstable flag (#4096) --- CHANGES.md | 17 ++ docs/faq.md | 7 +- docs/the_black_code_style/current_style.md | 6 + docs/the_black_code_style/future_style.md | 217 +++++++++--------- docs/the_black_code_style/index.md | 8 +- .../black_as_a_server.md | 6 + docs/usage_and_configuration/the_basics.md | 31 ++- pyproject.toml | 9 +- src/black/__init__.py | 49 +++- src/black/mode.py | 39 ++-- src/blackd/__init__.py | 104 +++++---- tests/data/cases/preview_cantfit.py | 14 -- tests/data/cases/preview_cantfit_string.py | 18 ++ tests/data/cases/preview_comments7.py | 2 +- tests/data/cases/preview_long_dict_values.py | 2 +- tests/data/cases/preview_long_strings.py | 2 +- .../preview_long_strings__east_asian_width.py | 2 +- .../cases/preview_long_strings__edge_case.py | 2 +- .../cases/preview_long_strings__regression.py | 2 +- tests/data/cases/preview_multiline_strings.py | 2 +- ...eview_return_annotation_brackets_string.py | 2 +- tests/test_black.py | 63 ++--- tests/util.py | 16 +- 23 files changed, 368 insertions(+), 252 deletions(-) create mode 100644 tests/data/cases/preview_cantfit_string.py diff --git a/CHANGES.md b/CHANGES.md index 45cb967e74a..0496603e2c0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -34,6 +34,14 @@ changes: - Fix incorrect formatting of certain async statements (#3609) - Allow combining `# fmt: skip` with other comments (#3959) +There are already a few improvements in the `--preview` style, which are slated for the +2025 stable style. Try them out and +[share your feedback](https://github.com/psf/black/issues). In the past, the preview +style has included some features that we were not able to stabilize. This year, we're +adding a separate `--unstable` style for features with known problems. Now, the +`--preview` style only includes features that we actually expect to make it into next +year's stable style. + ### Stable style @@ -53,6 +61,12 @@ release: +- Add `--unstable` style, covering preview features that have known problems that would + block them from going into the stable style. Also add the `--enable-unstable-feature` + flag; for example, use + `--enable-unstable-feature hug_parens_with_braces_and_square_brackets` to apply this + preview style throughout 2024, even if a later Black release downgrades the feature to + unstable (#4096) - Format module docstrings the same as class and function docstrings (#4095) - Fix crash when using a walrus in a dictionary (#4155) - Fix unnecessary parentheses when wrapping long dicts (#4135) @@ -66,6 +80,9 @@ release: - Fix symlink handling, properly catch and ignore symlinks that point outside of root (#4161) - Fix cache mtime logic that resulted in false positive cache hits (#4128) +- Remove the long-deprecated `--experimental-string-processing` flag. This feature can + currently be enabled with `--preview --enable-unstable-feature string_processing`. + (#4096) ### Packaging diff --git a/docs/faq.md b/docs/faq.md index c62e1b504b5..124a096efac 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -41,9 +41,10 @@ other tools, such as `# noqa`, may be moved by _Black_. See below for more detai Stable. _Black_ aims to enforce one style and one style only, with some room for pragmatism. See [The Black Code Style](the_black_code_style/index.md) for more details. -Starting in 2022, the formatting output will be stable for the releases made in the same -year (other than unintentional bugs). It is possible to opt-in to the latest formatting -styles, using the `--preview` flag. +Starting in 2022, the formatting output is stable for the releases made in the same year +(other than unintentional bugs). At the beginning of every year, the first release will +make changes to the stable style. It is possible to opt in to the latest formatting +styles using the `--preview` flag. ## Why is my file not formatted? diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index 00bd81416dc..ca5d1d4a701 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -449,6 +449,12 @@ file that are not enforced yet but might be in a future version of the formatter _Black_ will normalize line endings (`\n` or `\r\n`) based on the first line ending of the file. +### Form feed characters + +_Black_ will retain form feed characters on otherwise empty lines at the module level. +Only one form feed is retained for a group of consecutive empty lines. Where there are +two empty lines in a row, the form feed is placed on the second line. + ## Pragmatism Early versions of _Black_ used to be absolutist in some respects. They took after its diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index f55ea5f60a9..1cdd25fdb7c 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -1,54 +1,5 @@ # The (future of the) Black code style -```{warning} -Changes to this document often aren't tied and don't relate to releases of -_Black_. It's recommended that you read the latest version available. -``` - -## Using backslashes for with statements - -[Backslashes are bad and should be never be used](labels/why-no-backslashes) however -there is one exception: `with` statements using multiple context managers. Before Python -3.9 Python's grammar does not allow organizing parentheses around the series of context -managers. - -We don't want formatting like: - -```py3 -with make_context_manager1() as cm1, make_context_manager2() as cm2, make_context_manager3() as cm3, make_context_manager4() as cm4: - ... # nothing to split on - line too long -``` - -So _Black_ will, when we implement this, format it like this: - -```py3 -with \ - make_context_manager1() as cm1, \ - make_context_manager2() as cm2, \ - make_context_manager3() as cm3, \ - make_context_manager4() as cm4 \ -: - ... # backslashes and an ugly stranded colon -``` - -Although when the target version is Python 3.9 or higher, _Black_ uses parentheses -instead in `--preview` mode (see below) since they're allowed in Python 3.9 and higher. - -An alternative to consider if the backslashes in the above formatting are undesirable is -to use {external:py:obj}`contextlib.ExitStack` to combine context managers in the -following way: - -```python -with contextlib.ExitStack() as exit_stack: - cm1 = exit_stack.enter_context(make_context_manager1()) - cm2 = exit_stack.enter_context(make_context_manager2()) - cm3 = exit_stack.enter_context(make_context_manager3()) - cm4 = exit_stack.enter_context(make_context_manager4()) - ... -``` - -(labels/preview-style)= - ## Preview style Experimental, potentially disruptive style changes are gathered under the `--preview` @@ -56,62 +7,38 @@ CLI flag. At the end of each year, these changes may be adopted into the default as described in [The Black Code Style](index.md). Because the functionality is experimental, feedback and issue reports are highly encouraged! -### Improved string processing - -_Black_ will split long string literals and merge short ones. Parentheses are used where -appropriate. When split, parts of f-strings that don't need formatting are converted to -plain strings. User-made splits are respected when they do not exceed the line length -limit. Line continuation backslashes are converted into parenthesized strings. -Unnecessary parentheses are stripped. The stability and status of this feature is -tracked in [this issue](https://github.com/psf/black/issues/2188). - -### Improved line breaks - -For assignment expressions, _Black_ now prefers to split and wrap the right side of the -assignment instead of left side. For example: - -```python -some_dict[ - "with_a_long_key" -] = some_looooooooong_module.some_looooooooooooooong_function_name( - first_argument, second_argument, third_argument -) -``` - -will be changed to: +In the past, the preview style included some features with known bugs, so that we were +unable to move these features to the stable style. Therefore, such features are now +moved to the `--unstable` style. All features in the `--preview` style are expected to +make it to next year's stable style; features in the `--unstable` style will be +stabilized only if issues with them are fixed. If bugs are discovered in a `--preview` +feature, it is demoted to the `--unstable` style. To avoid thrash when a feature is +demoted from the `--preview` to the `--unstable` style, users can use the +`--enable-unstable-feature` flag to enable specific unstable features. -```python -some_dict["with_a_long_key"] = ( - some_looooooooong_module.some_looooooooooooooong_function_name( - first_argument, second_argument, third_argument - ) -) -``` +Currently, the following features are included in the preview style: -### Improved parentheses management +- `hex_codes_in_unicode_sequences`: normalize casing of Unicode escape characters in + strings +- `unify_docstring_detection`: fix inconsistencies in whether certain strings are + detected as docstrings +- `hug_parens_with_braces_and_square_brackets`: more compact formatting of nested + brackets ([see below](labels/hug-parens)) +- `no_normalize_fmt_skip_whitespace`: whitespace before `# fmt: skip` comments is no + longer normalized -For dict literals with long values, they are now wrapped in parentheses. Unnecessary -parentheses are now removed. For example: +(labels/unstable-features)= -```python -my_dict = { - "a key in my dict": a_very_long_variable - * and_a_very_long_function_call() - / 100000.0, - "another key": (short_value), -} -``` +The unstable style additionally includes the following features: -will be changed to: +- `string_processing`: split long string literals and related changes + ([see below](labels/string-processing)) +- `wrap_long_dict_values_in_parens`: add parentheses to long values in dictionaries + ([see below](labels/wrap-long-dict-values)) +- `multiline_string_handling`: more compact formatting of expressions involving + multiline strings ([see below](labels/multiline-string-handling)) -```python -my_dict = { - "a key in my dict": ( - a_very_long_variable * and_a_very_long_function_call() / 100000.0 - ), - "another key": short_value, -} -``` +(labels/hug-parens)= ### Improved multiline dictionary and list indentation for sole function parameter @@ -185,6 +112,46 @@ foo( ) ``` +(labels/string-processing)= + +### Improved string processing + +_Black_ will split long string literals and merge short ones. Parentheses are used where +appropriate. When split, parts of f-strings that don't need formatting are converted to +plain strings. User-made splits are respected when they do not exceed the line length +limit. Line continuation backslashes are converted into parenthesized strings. +Unnecessary parentheses are stripped. The stability and status of this feature is +tracked in [this issue](https://github.com/psf/black/issues/2188). + +(labels/wrap-long-dict-values)= + +### Improved parentheses management in dicts + +For dict literals with long values, they are now wrapped in parentheses. Unnecessary +parentheses are now removed. For example: + +```python +my_dict = { + "a key in my dict": a_very_long_variable + * and_a_very_long_function_call() + / 100000.0, + "another key": (short_value), +} +``` + +will be changed to: + +```python +my_dict = { + "a key in my dict": ( + a_very_long_variable * and_a_very_long_function_call() / 100000.0 + ), + "another key": short_value, +} +``` + +(labels/multiline-string-handling)= + ### Improved multiline string handling _Black_ is smarter when formatting multiline strings, especially in function arguments, @@ -297,13 +264,51 @@ s = ( # Top comment ) ``` -======= +## Potential future changes + +This section lists changes that we may want to make in the future, but that aren't +implemented yet. + +### Using backslashes for with statements + +[Backslashes are bad and should be never be used](labels/why-no-backslashes) however +there is one exception: `with` statements using multiple context managers. Before Python +3.9 Python's grammar does not allow organizing parentheses around the series of context +managers. + +We don't want formatting like: + +```py3 +with make_context_manager1() as cm1, make_context_manager2() as cm2, make_context_manager3() as cm3, make_context_manager4() as cm4: + ... # nothing to split on - line too long +``` + +So _Black_ will, when we implement this, format it like this: + +```py3 +with \ + make_context_manager1() as cm1, \ + make_context_manager2() as cm2, \ + make_context_manager3() as cm3, \ + make_context_manager4() as cm4 \ +: + ... # backslashes and an ugly stranded colon +``` + +Although when the target version is Python 3.9 or higher, _Black_ uses parentheses +instead in `--preview` mode (see below) since they're allowed in Python 3.9 and higher. -### Form feed characters +An alternative to consider if the backslashes in the above formatting are undesirable is +to use {external:py:obj}`contextlib.ExitStack` to combine context managers in the +following way: -_Black_ will now retain form feed characters on otherwise empty lines at the module -level. Only one form feed is retained for a group of consecutive empty lines. Where -there are two empty lines in a row, the form feed will be placed on the second line. +```python +with contextlib.ExitStack() as exit_stack: + cm1 = exit_stack.enter_context(make_context_manager1()) + cm2 = exit_stack.enter_context(make_context_manager2()) + cm3 = exit_stack.enter_context(make_context_manager3()) + cm4 = exit_stack.enter_context(make_context_manager4()) + ... +``` -_Black_ already retained form feed literals inside a comment or inside a string. This -remains the case. +(labels/preview-style)= diff --git a/docs/the_black_code_style/index.md b/docs/the_black_code_style/index.md index 1719347eec8..58f28673022 100644 --- a/docs/the_black_code_style/index.md +++ b/docs/the_black_code_style/index.md @@ -42,9 +42,11 @@ _Black_: enabled by newer Python language syntax as well as due to improvements in the formatting logic. -- The `--preview` flag is exempt from this policy. There are no guarantees around the - stability of the output with that flag passed into _Black_. This flag is intended for - allowing experimentation with the proposed changes to the _Black_ code style. +- The `--preview` and `--unstable` flags are exempt from this policy. There are no + guarantees around the stability of the output with these flags passed into _Black_. + They are intended for allowing experimentation with proposed changes to the _Black_ + code style. The `--preview` style at the end of a year should closely match the stable + style for the next year, but we may always make changes. Documentation for both the current and future styles can be found: diff --git a/docs/usage_and_configuration/black_as_a_server.md b/docs/usage_and_configuration/black_as_a_server.md index f24fb34d915..0a7edb57fd7 100644 --- a/docs/usage_and_configuration/black_as_a_server.md +++ b/docs/usage_and_configuration/black_as_a_server.md @@ -62,6 +62,12 @@ The headers controlling how source code is formatted are: - `X-Preview`: corresponds to the `--preview` command line flag. If present and its value is not an empty string, experimental and potentially disruptive style changes will be used. +- `X-Unstable`: corresponds to the `--unstable` command line flag. If present and its + value is not an empty string, experimental style changes that are known to be buggy + will be used. +- `X-Enable-Unstable-Feature`: corresponds to the `--enable-unstable-feature` flag. The + contents of the flag must be a comma-separated list of unstable features to be + enabled. Example: `X-Enable-Unstable-Feature: feature1, feature2`. - `X-Fast-Or-Safe`: if set to `fast`, `blackd` will act as _Black_ does when passed the `--fast` command line flag. - `X-Python-Variant`: if set to `pyi`, `blackd` will act as _Black_ does when passed the diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index b541f07907c..a42e093155b 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -144,9 +144,34 @@ magic trailing comma is ignored. #### `--preview` -Enable potentially disruptive style changes that may be added to Black's main -functionality in the next major release. Read more about -[our preview style](labels/preview-style). +Enable potentially disruptive style changes that we expect to add to Black's main +functionality in the next major release. Use this if you want a taste of what next +year's style will look like. + +Read more about [our preview style](labels/preview-style). + +There is no guarantee on the code style produced by this flag across releases. + +#### `--unstable` + +Enable all style changes in `--preview`, plus additional changes that we would like to +make eventually, but that have known issues that need to be fixed before they can move +back to the `--preview` style. Use this if you want to experiment with these changes and +help fix issues with them. + +There is no guarantee on the code style produced by this flag across releases. + +#### `--enable-unstable-feature` + +Enable specific features from the `--unstable` style. See +[the preview style documentation](labels/unstable-features) for the list of supported +features. This flag can only be used when `--preview` is enabled. Users are encouraged +to use this flag if they use `--preview` style and a feature that affects their code is +moved from the `--preview` to the `--unstable` style, but they want to avoid the thrash +from undoing this change. + +There are no guarantees on the behavior of these features, or even their existence, +across releases. (labels/exit-code)= diff --git a/pyproject.toml b/pyproject.toml index 24b9c07674d..fa3654b8d67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,10 +16,11 @@ extend-exclude = ''' | profiling )/ ''' -# We use preview style for formatting Black itself. If you -# want stable formatting across releases, you should keep -# this off. -preview = true +# We use the unstable style for formatting Black itself. If you +# want bug-free formatting, you should keep this off. If you want +# stable formatting across releases, you should also keep `preview = true` +# (which is implied by this flag) off. +unstable = true # Build system information and other project-specific configuration below. # NOTE: You don't need this in your own Black configuration. diff --git a/src/black/__init__.py b/src/black/__init__.py index 961ed9479a8..ebc7ac8eda5 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -68,7 +68,7 @@ from black.lines import EmptyLineTracker, LinesBlock from black.mode import FUTURE_FLAG_TO_FEATURE, VERSION_TO_FEATURES, Feature from black.mode import Mode as Mode # re-exported -from black.mode import TargetVersion, supports_feature +from black.mode import Preview, TargetVersion, supports_feature from black.nodes import ( STARS, is_number_token, @@ -209,6 +209,13 @@ def target_version_option_callback( return [TargetVersion[val.upper()] for val in v] +def enable_unstable_feature_callback( + c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...] +) -> List[Preview]: + """Compute the features from an --enable-unstable-feature flag.""" + return [Preview[val] for val in v] + + def re_compile_maybe_verbose(regex: str) -> Pattern[str]: """Compile a regular expression string in `regex`. @@ -303,12 +310,6 @@ def validate_regex( is_flag=True, help="Don't use trailing commas as a reason to split lines.", ) -@click.option( - "--experimental-string-processing", - is_flag=True, - hidden=True, - help="(DEPRECATED and now included in --preview) Normalize string literals.", -) @click.option( "--preview", is_flag=True, @@ -317,6 +318,26 @@ def validate_regex( " functionality in the next major release." ), ) +@click.option( + "--unstable", + is_flag=True, + help=( + "Enable potentially disruptive style changes that have known bugs or are not" + " currently expected to make it into the stable style Black's next major" + " release. Implies --preview." + ), +) +@click.option( + "--enable-unstable-feature", + type=click.Choice([v.name for v in Preview]), + callback=enable_unstable_feature_callback, + multiple=True, + help=( + "Enable specific features included in the `--unstable` style. Requires" + " `--preview`. No compatibility guarantees are provided on the behavior" + " or existence of any unstable features." + ), +) @click.option( "--check", is_flag=True, @@ -507,8 +528,9 @@ def main( # noqa: C901 skip_source_first_line: bool, skip_string_normalization: bool, skip_magic_trailing_comma: bool, - experimental_string_processing: bool, preview: bool, + unstable: bool, + enable_unstable_feature: List[Preview], quiet: bool, verbose: bool, required_version: Optional[str], @@ -534,6 +556,14 @@ def main( # noqa: C901 out(main.get_usage(ctx) + "\n\nOne of 'SRC' or 'code' is required.") ctx.exit(1) + # It doesn't do anything if --unstable is also passed, so just allow it. + if enable_unstable_feature and not (preview or unstable): + out( + main.get_usage(ctx) + + "\n\n'--enable-unstable-feature' requires '--preview'." + ) + ctx.exit(1) + root, method = ( find_project_root(src, stdin_filename) if code is None else (None, None) ) @@ -595,9 +625,10 @@ def main( # noqa: C901 skip_source_first_line=skip_source_first_line, string_normalization=not skip_string_normalization, magic_trailing_comma=not skip_magic_trailing_comma, - experimental_string_processing=experimental_string_processing, preview=preview, + unstable=unstable, python_cell_magics=set(python_cell_magics), + enabled_features=set(enable_unstable_feature), ) lines: List[Tuple[int, int]] = [] diff --git a/src/black/mode.py b/src/black/mode.py index a1519f17bcc..68919fb4901 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -9,7 +9,6 @@ from hashlib import sha256 from operator import attrgetter from typing import Dict, Final, Set -from warnings import warn from black.const import DEFAULT_LINE_LENGTH @@ -179,6 +178,16 @@ class Preview(Enum): multiline_string_handling = auto() +UNSTABLE_FEATURES: Set[Preview] = { + # Many issues, see summary in https://github.com/psf/black/issues/4042 + Preview.string_processing, + # See issues #3452 and #4158 + Preview.wrap_long_dict_values_in_parens, + # See issue #4159 + Preview.multiline_string_handling, +} + + class Deprecated(UserWarning): """Visible deprecation warning.""" @@ -192,28 +201,24 @@ class Mode: is_ipynb: bool = False skip_source_first_line: bool = False magic_trailing_comma: bool = True - experimental_string_processing: bool = False python_cell_magics: Set[str] = field(default_factory=set) preview: bool = False - - def __post_init__(self) -> None: - if self.experimental_string_processing: - warn( - "`experimental string processing` has been included in `preview`" - " and deprecated. Use `preview` instead.", - Deprecated, - ) + unstable: bool = False + enabled_features: Set[Preview] = field(default_factory=set) def __contains__(self, feature: Preview) -> bool: """ Provide `Preview.FEATURE in Mode` syntax that mirrors the ``preview`` flag. - The argument is not checked and features are not differentiated. - They only exist to make development easier by clarifying intent. + In unstable mode, all features are enabled. In preview mode, all features + except those in UNSTABLE_FEATURES are enabled. Any features in + `self.enabled_features` are also enabled. """ - if feature is Preview.string_processing: - return self.preview or self.experimental_string_processing - return self.preview + if self.unstable: + return True + if feature in self.enabled_features: + return True + return self.preview and feature not in UNSTABLE_FEATURES def get_cache_key(self) -> str: if self.target_versions: @@ -231,7 +236,9 @@ def get_cache_key(self) -> str: str(int(self.is_ipynb)), str(int(self.skip_source_first_line)), str(int(self.magic_trailing_comma)), - str(int(self.experimental_string_processing)), + sha256( + (",".join(sorted(f.name for f in self.enabled_features))).encode() + ).hexdigest(), str(int(self.preview)), sha256((",".join(sorted(self.python_cell_magics))).encode()).hexdigest(), ] diff --git a/src/blackd/__init__.py b/src/blackd/__init__.py index 6b0f3d33295..7041671f596 100644 --- a/src/blackd/__init__.py +++ b/src/blackd/__init__.py @@ -8,6 +8,7 @@ try: from aiohttp import web + from multidict import MultiMapping from .middlewares import cors except ImportError as ie: @@ -34,6 +35,8 @@ SKIP_STRING_NORMALIZATION_HEADER = "X-Skip-String-Normalization" SKIP_MAGIC_TRAILING_COMMA = "X-Skip-Magic-Trailing-Comma" PREVIEW = "X-Preview" +UNSTABLE = "X-Unstable" +ENABLE_UNSTABLE_FEATURE = "X-Enable-Unstable-Feature" FAST_OR_SAFE_HEADER = "X-Fast-Or-Safe" DIFF_HEADER = "X-Diff" @@ -45,6 +48,8 @@ SKIP_STRING_NORMALIZATION_HEADER, SKIP_MAGIC_TRAILING_COMMA, PREVIEW, + UNSTABLE, + ENABLE_UNSTABLE_FEATURE, FAST_OR_SAFE_HEADER, DIFF_HEADER, ] @@ -53,6 +58,10 @@ BLACK_VERSION_HEADER = "X-Black-Version" +class HeaderError(Exception): + pass + + class InvalidVariantHeader(Exception): pass @@ -93,55 +102,21 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: return web.Response( status=501, text="This server only supports protocol version 1" ) - try: - line_length = int( - request.headers.get(LINE_LENGTH_HEADER, black.DEFAULT_LINE_LENGTH) - ) - except ValueError: - return web.Response(status=400, text="Invalid line length header value") - if PYTHON_VARIANT_HEADER in request.headers: - value = request.headers[PYTHON_VARIANT_HEADER] - try: - pyi, versions = parse_python_variant_header(value) - except InvalidVariantHeader as e: - return web.Response( - status=400, - text=f"Invalid value for {PYTHON_VARIANT_HEADER}: {e.args[0]}", - ) - else: - pyi = False - versions = set() - - skip_string_normalization = bool( - request.headers.get(SKIP_STRING_NORMALIZATION_HEADER, False) - ) - skip_magic_trailing_comma = bool( - request.headers.get(SKIP_MAGIC_TRAILING_COMMA, False) - ) - skip_source_first_line = bool( - request.headers.get(SKIP_SOURCE_FIRST_LINE, False) - ) - preview = bool(request.headers.get(PREVIEW, False)) fast = False if request.headers.get(FAST_OR_SAFE_HEADER, "safe") == "fast": fast = True - mode = black.FileMode( - target_versions=versions, - is_pyi=pyi, - line_length=line_length, - skip_source_first_line=skip_source_first_line, - string_normalization=not skip_string_normalization, - magic_trailing_comma=not skip_magic_trailing_comma, - preview=preview, - ) + try: + mode = parse_mode(request.headers) + except HeaderError as e: + return web.Response(status=400, text=e.args[0]) req_bytes = await request.content.read() charset = request.charset if request.charset is not None else "utf8" req_str = req_bytes.decode(charset) then = datetime.now(timezone.utc) header = "" - if skip_source_first_line: + if mode.skip_source_first_line: first_newline_position: int = req_str.find("\n") + 1 header = req_str[:first_newline_position] req_str = req_str[first_newline_position:] @@ -190,6 +165,57 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: return web.Response(status=500, headers=headers, text=str(e)) +def parse_mode(headers: MultiMapping[str]) -> black.Mode: + try: + line_length = int(headers.get(LINE_LENGTH_HEADER, black.DEFAULT_LINE_LENGTH)) + except ValueError: + raise HeaderError("Invalid line length header value") from None + + if PYTHON_VARIANT_HEADER in headers: + value = headers[PYTHON_VARIANT_HEADER] + try: + pyi, versions = parse_python_variant_header(value) + except InvalidVariantHeader as e: + raise HeaderError( + f"Invalid value for {PYTHON_VARIANT_HEADER}: {e.args[0]}", + ) from None + else: + pyi = False + versions = set() + + skip_string_normalization = bool( + headers.get(SKIP_STRING_NORMALIZATION_HEADER, False) + ) + skip_magic_trailing_comma = bool(headers.get(SKIP_MAGIC_TRAILING_COMMA, False)) + skip_source_first_line = bool(headers.get(SKIP_SOURCE_FIRST_LINE, False)) + + preview = bool(headers.get(PREVIEW, False)) + unstable = bool(headers.get(UNSTABLE, False)) + enable_features: Set[black.Preview] = set() + enable_unstable_features = headers.get(ENABLE_UNSTABLE_FEATURE, "").split(",") + for piece in enable_unstable_features: + piece = piece.strip() + if piece: + try: + enable_features.add(black.Preview[piece]) + except KeyError: + raise HeaderError( + f"Invalid value for {ENABLE_UNSTABLE_FEATURE}: {piece}", + ) from None + + return black.FileMode( + target_versions=versions, + is_pyi=pyi, + line_length=line_length, + skip_source_first_line=skip_source_first_line, + string_normalization=not skip_string_normalization, + magic_trailing_comma=not skip_magic_trailing_comma, + preview=preview, + unstable=unstable, + enabled_features=enable_features, + ) + + def parse_python_variant_header(value: str) -> Tuple[bool, Set[black.TargetVersion]]: if value == "pyi": return True, set() diff --git a/tests/data/cases/preview_cantfit.py b/tests/data/cases/preview_cantfit.py index d5da6654f0c..29789c7e653 100644 --- a/tests/data/cases/preview_cantfit.py +++ b/tests/data/cases/preview_cantfit.py @@ -20,12 +20,6 @@ normal_name = but_the_function_name_is_now_ridiculously_long_and_it_is_still_super_annoying( [1, 2, 3], arg1, [1, 2, 3], arg2, [1, 2, 3], arg3 ) -# long arguments -normal_name = normal_function_name( - "but with super long string arguments that on their own exceed the line limit so there's no way it can ever fit", - "eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs", - this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it=0, -) string_variable_name = ( "a string that is waaaaaaaayyyyyyyy too long, even in parens, there's nothing you can do" # noqa ) @@ -78,14 +72,6 @@ [1, 2, 3], arg1, [1, 2, 3], arg2, [1, 2, 3], arg3 ) ) -# long arguments -normal_name = normal_function_name( - "but with super long string arguments that on their own exceed the line limit so" - " there's no way it can ever fit", - "eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs" - " with spam and eggs and spam with eggs", - this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it=0, -) string_variable_name = "a string that is waaaaaaaayyyyyyyy too long, even in parens, there's nothing you can do" # noqa for key in """ hostname diff --git a/tests/data/cases/preview_cantfit_string.py b/tests/data/cases/preview_cantfit_string.py new file mode 100644 index 00000000000..3b48e318ade --- /dev/null +++ b/tests/data/cases/preview_cantfit_string.py @@ -0,0 +1,18 @@ +# flags: --unstable +# long arguments +normal_name = normal_function_name( + "but with super long string arguments that on their own exceed the line limit so there's no way it can ever fit", + "eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs", + this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it=0, +) + +# output + +# long arguments +normal_name = normal_function_name( + "but with super long string arguments that on their own exceed the line limit so" + " there's no way it can ever fit", + "eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs" + " with spam and eggs and spam with eggs", + this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it=0, +) diff --git a/tests/data/cases/preview_comments7.py b/tests/data/cases/preview_comments7.py index 006d4f7266f..e4d547138db 100644 --- a/tests/data/cases/preview_comments7.py +++ b/tests/data/cases/preview_comments7.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable from .config import ( Any, Bool, diff --git a/tests/data/cases/preview_long_dict_values.py b/tests/data/cases/preview_long_dict_values.py index 54da76038dc..a19210605f6 100644 --- a/tests/data/cases/preview_long_dict_values.py +++ b/tests/data/cases/preview_long_dict_values.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable my_dict = { "something_something": r"Lorem ipsum dolor sit amet, an sed convenire eloquentiam \t" diff --git a/tests/data/cases/preview_long_strings.py b/tests/data/cases/preview_long_strings.py index 19ac47a7032..86fa1b0c7e1 100644 --- a/tests/data/cases/preview_long_strings.py +++ b/tests/data/cases/preview_long_strings.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable x = "This is a really long string that can't possibly be expected to fit all together on one line. In fact it may even take up three or more lines... like four or five... but probably just three." x += "This is a really long string that can't possibly be expected to fit all together on one line. In fact it may even take up three or more lines... like four or five... but probably just three." diff --git a/tests/data/cases/preview_long_strings__east_asian_width.py b/tests/data/cases/preview_long_strings__east_asian_width.py index d190f422a60..022b0452522 100644 --- a/tests/data/cases/preview_long_strings__east_asian_width.py +++ b/tests/data/cases/preview_long_strings__east_asian_width.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable # The following strings do not have not-so-many chars, but are long enough # when these are rendered in a monospace font (if the renderer respects # Unicode East Asian Width properties). diff --git a/tests/data/cases/preview_long_strings__edge_case.py b/tests/data/cases/preview_long_strings__edge_case.py index a8e8971968c..28497e731bc 100644 --- a/tests/data/cases/preview_long_strings__edge_case.py +++ b/tests/data/cases/preview_long_strings__edge_case.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable some_variable = "This string is long but not so long that it needs to be split just yet" some_variable = 'This string is long but not so long that it needs to be split just yet' some_variable = "This string is long, just long enough that it needs to be split, u get?" diff --git a/tests/data/cases/preview_long_strings__regression.py b/tests/data/cases/preview_long_strings__regression.py index 5e76a8cf61c..afe2b311cf4 100644 --- a/tests/data/cases/preview_long_strings__regression.py +++ b/tests/data/cases/preview_long_strings__regression.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable class A: def foo(): result = type(message)("") diff --git a/tests/data/cases/preview_multiline_strings.py b/tests/data/cases/preview_multiline_strings.py index 3ff643610b7..9288f6991bd 100644 --- a/tests/data/cases/preview_multiline_strings.py +++ b/tests/data/cases/preview_multiline_strings.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable """cow say""", call(3, "dogsay", textwrap.dedent("""dove diff --git a/tests/data/cases/preview_return_annotation_brackets_string.py b/tests/data/cases/preview_return_annotation_brackets_string.py index fea0ea6839a..2f937cf54ef 100644 --- a/tests/data/cases/preview_return_annotation_brackets_string.py +++ b/tests/data/cases/preview_return_annotation_brackets_string.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable # Long string example def frobnicate() -> "ThisIsTrulyUnreasonablyExtremelyLongClassName | list[ThisIsTrulyUnreasonablyExtremelyLongClassName]": pass diff --git a/tests/test_black.py b/tests/test_black.py index a979a95b674..6dbe25a90b6 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -158,13 +158,11 @@ def test_empty_ff(self) -> None: @patch("black.dump_to_file", dump_to_stderr) def test_one_empty_line(self) -> None: - mode = black.Mode(preview=True) for nl in ["\n", "\r\n"]: source = expected = nl - assert_format(source, expected, mode=mode) + assert_format(source, expected) def test_one_empty_line_ff(self) -> None: - mode = black.Mode(preview=True) for nl in ["\n", "\r\n"]: expected = nl tmp_file = Path(black.dump_to_file(nl)) @@ -175,20 +173,13 @@ def test_one_empty_line_ff(self) -> None: with open(tmp_file, "wb") as f: f.write(nl.encode("utf-8")) try: - self.assertFalse( - ff(tmp_file, mode=mode, write_back=black.WriteBack.YES) - ) + self.assertFalse(ff(tmp_file, write_back=black.WriteBack.YES)) with open(tmp_file, "rb") as f: actual = f.read().decode("utf-8") finally: os.unlink(tmp_file) self.assertFormatEqual(expected, actual) - def test_experimental_string_processing_warns(self) -> None: - self.assertWarns( - black.mode.Deprecated, black.Mode, experimental_string_processing=True - ) - def test_piping(self) -> None: _, source, expected = read_data_from_file( PROJECT_ROOT / "src/black/__init__.py" @@ -252,21 +243,6 @@ def test_piping_diff_with_color(self) -> None: self.assertIn("\033[31m", actual) self.assertIn("\033[0m", actual) - @patch("black.dump_to_file", dump_to_stderr) - def _test_wip(self) -> None: - source, expected = read_data("miscellaneous", "wip") - sys.settrace(tracefunc) - mode = replace( - DEFAULT_MODE, - experimental_string_processing=False, - target_versions={black.TargetVersion.PY38}, - ) - actual = fs(source, mode=mode) - sys.settrace(None) - self.assertFormatEqual(expected, actual) - black.assert_equivalent(source, actual) - black.assert_stable(source, actual, black.FileMode()) - def test_pep_572_version_detection(self) -> None: source, _ = read_data("cases", "pep_572") root = black.lib2to3_parse(source) @@ -374,7 +350,7 @@ def test_detect_debug_f_strings(self) -> None: @patch("black.dump_to_file", dump_to_stderr) def test_string_quotes(self) -> None: source, expected = read_data("miscellaneous", "string_quotes") - mode = black.Mode(preview=True) + mode = black.Mode(unstable=True) assert_format(source, expected, mode) mode = replace(mode, string_normalization=False) not_normalized = fs(source, mode=mode) @@ -1052,7 +1028,6 @@ def test_format_file_contents(self) -> None: black.format_file_contents(invalid, mode=mode, fast=False) self.assertEqual(str(e.exception), "Cannot parse: 1:7: return if you can") - mode = black.Mode(preview=True) just_crlf = "\r\n" with self.assertRaises(black.NothingChanged): black.format_file_contents(just_crlf, mode=mode, fast=False) @@ -1396,7 +1371,6 @@ def get_output(*args: Any, **kwargs: Any) -> io.TextIOWrapper: return get_output - mode = black.Mode(preview=True) for content, expected in cases: output = io.StringIO() io_TextIOWrapper = io.TextIOWrapper @@ -1407,26 +1381,27 @@ def get_output(*args: Any, **kwargs: Any) -> io.TextIOWrapper: fast=True, content=content, write_back=black.WriteBack.YES, - mode=mode, + mode=DEFAULT_MODE, ) except io.UnsupportedOperation: pass # StringIO does not support detach assert output.getvalue() == expected - # An empty string is the only test case for `preview=False` - output = io.StringIO() - io_TextIOWrapper = io.TextIOWrapper - with patch("io.TextIOWrapper", _new_wrapper(output, io_TextIOWrapper)): - try: - black.format_stdin_to_stdout( - fast=True, - content="", - write_back=black.WriteBack.YES, - mode=DEFAULT_MODE, - ) - except io.UnsupportedOperation: - pass # StringIO does not support detach - assert output.getvalue() == "" + def test_cli_unstable(self) -> None: + self.invokeBlack(["--unstable", "-c", "0"], exit_code=0) + self.invokeBlack(["--preview", "-c", "0"], exit_code=0) + # Must also pass --preview + self.invokeBlack( + ["--enable-unstable-feature", "string_processing", "-c", "0"], exit_code=1 + ) + self.invokeBlack( + ["--preview", "--enable-unstable-feature", "string_processing", "-c", "0"], + exit_code=0, + ) + self.invokeBlack( + ["--unstable", "--enable-unstable-feature", "string_processing", "-c", "0"], + exit_code=0, + ) def test_invalid_cli_regex(self) -> None: for option in ["--include", "--exclude", "--extend-exclude", "--force-exclude"]: diff --git a/tests/util.py b/tests/util.py index 9ea30e62fe3..d5425f1f743 100644 --- a/tests/util.py +++ b/tests/util.py @@ -112,16 +112,24 @@ def assert_format( # For both preview and non-preview tests, ensure that Black doesn't crash on # this code, but don't pass "expected" because the precise output may differ. try: + if mode.unstable: + new_mode = replace(mode, unstable=False, preview=False) + else: + new_mode = replace(mode, preview=not mode.preview) _assert_format_inner( source, None, - replace(mode, preview=not mode.preview), + new_mode, fast=fast, minimum_version=minimum_version, lines=lines, ) except Exception as e: - text = "non-preview" if mode.preview else "preview" + text = ( + "unstable" + if mode.unstable + else "non-preview" if mode.preview else "preview" + ) raise FormatFailure( f"Black crashed formatting this case in {text} mode." ) from e @@ -138,7 +146,7 @@ def assert_format( _assert_format_inner( source, None, - replace(mode, preview=preview_mode, line_length=1), + replace(mode, preview=preview_mode, line_length=1, unstable=False), fast=fast, minimum_version=minimum_version, lines=lines, @@ -241,6 +249,7 @@ def get_flags_parser() -> argparse.ArgumentParser: "--skip-magic-trailing-comma", default=False, action="store_true" ) parser.add_argument("--preview", default=False, action="store_true") + parser.add_argument("--unstable", default=False, action="store_true") parser.add_argument("--fast", default=False, action="store_true") parser.add_argument( "--minimum-version", @@ -278,6 +287,7 @@ def parse_mode(flags_line: str) -> TestCaseArgs: is_ipynb=args.ipynb, magic_trailing_comma=not args.skip_magic_trailing_comma, preview=args.preview, + unstable=args.unstable, ) if args.line_ranges: lines = parse_line_ranges(args.line_ranges) From 0e6e46b9eb45f5a22062fe84c2c2ff46bd0d738e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 25 Jan 2024 20:35:21 -0800 Subject: [PATCH 27/68] Prepare release 24.1.0 (#4170) --- CHANGES.md | 46 +++------------------ docs/integrations/source_version_control.md | 4 +- docs/usage_and_configuration/the_basics.md | 6 +-- scripts/release.py | 4 +- 4 files changed, 13 insertions(+), 47 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0496603e2c0..ff921de69eb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,11 +1,9 @@ # Change Log -## Unreleased +## 24.1.0 ### Highlights - - This release introduces the new 2024 stable style (#4106), stabilizing the following changes: @@ -44,8 +42,6 @@ year's stable style. ### Stable style - - Several bug fixes were made in features that are moved to the stable style in this release: @@ -59,14 +55,12 @@ release: ### Preview style - - - Add `--unstable` style, covering preview features that have known problems that would block them from going into the stable style. Also add the `--enable-unstable-feature` flag; for example, use `--enable-unstable-feature hug_parens_with_braces_and_square_brackets` to apply this - preview style throughout 2024, even if a later Black release downgrades the feature to - unstable (#4096) + preview feature throughout 2024, even if a later Black release downgrades the feature + to unstable (#4096) - Format module docstrings the same as class and function docstrings (#4095) - Fix crash when using a walrus in a dictionary (#4155) - Fix unnecessary parentheses when wrapping long dicts (#4135) @@ -74,48 +68,18 @@ release: ### Configuration - - -- Print warning when toml config contains an invalid key (#4165) -- Fix symlink handling, properly catch and ignore symlinks that point outside of root - (#4161) +- Print warning when configuration in `pyproject.toml` contains an invalid key (#4165) +- Fix symlink handling, properly ignoring symlinks that point outside of root (#4161) - Fix cache mtime logic that resulted in false positive cache hits (#4128) - Remove the long-deprecated `--experimental-string-processing` flag. This feature can currently be enabled with `--preview --enable-unstable-feature string_processing`. (#4096) -### Packaging - - - -### Parser - - - -### Performance - - - -### Output - - - -### _Blackd_ - - - ### Integrations - - - Revert the change to run Black's pre-commit integration only on specific git hooks (#3940) for better compatibility with older versions of pre-commit (#4137) -### Documentation - - - ## 23.12.1 ### Packaging diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index 3b895193941..259c1c1eaf3 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -8,7 +8,7 @@ Use [pre-commit](https://pre-commit.com/). Once you repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.12.1 + rev: 24.1.0 hooks: - id: black # It is recommended to specify the latest version of Python @@ -35,7 +35,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.12.1 + rev: 24.1.0 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index a42e093155b..562fd7d5905 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -266,8 +266,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 23.12.1 (compiled: yes) -$ black --required-version 23.12.1 -c "format = 'this'" +black, 24.1.0 (compiled: yes) +$ black --required-version 24.1.0 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -363,7 +363,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 23.12.1 +black, 24.1.0 ``` #### `--config` diff --git a/scripts/release.py b/scripts/release.py index d588429c2d3..c5336506396 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -169,7 +169,9 @@ def get_next_version(self) -> str: calver_parts = base_calver.split(".") base_calver = f"{calver_parts[0]}.{int(calver_parts[1])}" # Remove leading 0 git_tags = get_git_tags() - same_month_releases = [t for t in git_tags if t.startswith(base_calver)] + same_month_releases = [ + t for t in git_tags if t.startswith(base_calver) and "a" not in t + ] if len(same_month_releases) < 1: return f"{base_calver}.0" same_month_version = same_month_releases[-1].split(".", 2)[-1] From 659c29a41c7c686687aef21f57b95bcfa236b03b Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 25 Jan 2024 21:12:38 -0800 Subject: [PATCH 28/68] New changelog --- CHANGES.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index ff921de69eb..9a9be4bbeae 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,52 @@ # Change Log +## Unreleased + +### Highlights + + + +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + + ## 24.1.0 ### Highlights From ed770ba4dd50c419148a0fca2b43937a7447e1f9 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 26 Jan 2024 11:54:49 -0800 Subject: [PATCH 29/68] Fix cache file length (#4176) - Ensure total file length stays under 96 - Hash the path only if it's too long - Proceed normally (with a warning) if the cache can't be read Fixes #4172 --- CHANGES.md | 3 +++ src/black/cache.py | 9 ++++++++- src/black/mode.py | 21 +++++++++++++++++---- tests/test_black.py | 25 +++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9a9be4bbeae..e4240eacfca 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,6 +18,9 @@ +- Shorten the length of the name of the cache file to fix crashes on file systems that + do not support long paths (#4176) + ### Packaging diff --git a/src/black/cache.py b/src/black/cache.py index cfdbc21e92a..35bddb573d2 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -13,6 +13,7 @@ from _black_version import version as __version__ from black.mode import Mode +from black.output import err if sys.version_info >= (3, 11): from typing import Self @@ -64,7 +65,13 @@ def read(cls, mode: Mode) -> Self: resolve the issue. """ cache_file = get_cache_file(mode) - if not cache_file.exists(): + try: + exists = cache_file.exists() + except OSError as e: + # Likely file too long; see #4172 and #4174 + err(f"Unable to read cache file {cache_file} due to {e}") + return cls(mode, cache_file) + if not exists: return cls(mode, cache_file) with cache_file.open("rb") as fobj: diff --git a/src/black/mode.py b/src/black/mode.py index 68919fb4901..128d2b9f108 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -192,6 +192,9 @@ class Deprecated(UserWarning): """Visible deprecation warning.""" +_MAX_CACHE_KEY_PART_LENGTH: Final = 32 + + @dataclass class Mode: target_versions: Set[TargetVersion] = field(default_factory=set) @@ -228,6 +231,19 @@ def get_cache_key(self) -> str: ) else: version_str = "-" + if len(version_str) > _MAX_CACHE_KEY_PART_LENGTH: + version_str = sha256(version_str.encode()).hexdigest()[ + :_MAX_CACHE_KEY_PART_LENGTH + ] + features_and_magics = ( + ",".join(sorted(f.name for f in self.enabled_features)) + + "@" + + ",".join(sorted(self.python_cell_magics)) + ) + if len(features_and_magics) > _MAX_CACHE_KEY_PART_LENGTH: + features_and_magics = sha256(features_and_magics.encode()).hexdigest()[ + :_MAX_CACHE_KEY_PART_LENGTH + ] parts = [ version_str, str(self.line_length), @@ -236,10 +252,7 @@ def get_cache_key(self) -> str: str(int(self.is_ipynb)), str(int(self.skip_source_first_line)), str(int(self.magic_trailing_comma)), - sha256( - (",".join(sorted(f.name for f in self.enabled_features))).encode() - ).hexdigest(), str(int(self.preview)), - sha256((",".join(sorted(self.python_cell_magics))).encode()).hexdigest(), + features_and_magics, ] return ".".join(parts) diff --git a/tests/test_black.py b/tests/test_black.py index 6dbe25a90b6..123ea0bb88a 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -44,6 +44,7 @@ from black import re_compile_maybe_verbose as compile_pattern from black.cache import FileData, get_cache_dir, get_cache_file from black.debug import DebugVisitor +from black.mode import Mode, Preview from black.output import color_diff, diff from black.report import Report @@ -2065,6 +2066,30 @@ def test_get_cache_dir( monkeypatch.setenv("BLACK_CACHE_DIR", str(workspace2)) assert get_cache_dir().parent == workspace2 + def test_cache_file_length(self) -> None: + cases = [ + DEFAULT_MODE, + # all of the target versions + Mode(target_versions=set(TargetVersion)), + # all of the features + Mode(enabled_features=set(Preview)), + # all of the magics + Mode(python_cell_magics={f"magic{i}" for i in range(500)}), + # all of the things + Mode( + target_versions=set(TargetVersion), + enabled_features=set(Preview), + python_cell_magics={f"magic{i}" for i in range(500)}, + ), + ] + for case in cases: + cache_file = get_cache_file(case) + # Some common file systems enforce a maximum path length + # of 143 (issue #4174). We can't do anything if the directory + # path is too long, but ensure the name of the cache file itself + # doesn't get too crazy. + assert len(cache_file.name) <= 96 + def test_cache_broken_file(self) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: From 1607e9ab20ad550cf940482d0d361ca31fc03189 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 27 Jan 2024 12:34:02 -0800 Subject: [PATCH 30/68] Fix missing space in option description (#4182) --- src/black/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index ebc7ac8eda5..8ab5b47f974 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -279,7 +279,7 @@ def validate_regex( is_flag=True, help=( "Format all input files like Jupyter Notebooks regardless of file extension." - "This is useful when piping source on standard input." + " This is useful when piping source on standard input." ), ) @click.option( From 8bf04549ffd276a1bad6eb110e66e6557ee630d9 Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Sat, 27 Jan 2024 15:55:22 -0600 Subject: [PATCH 31/68] Consistently add trailing comma on typed parameters (#4164) Signed-off-by: RedGuy12 <61329810+RedGuy12@users.noreply.github.com> Co-authored-by: Jelle Zijlstra --- CHANGES.md | 2 ++ docs/the_black_code_style/future_style.md | 2 ++ src/black/files.py | 2 +- src/black/linegen.py | 10 +++++++- src/black/mode.py | 1 + src/blib2to3/pgen2/parse.py | 2 +- .../data/cases/typed_params_trailing_comma.py | 24 +++++++++++++++++++ 7 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 tests/data/cases/typed_params_trailing_comma.py diff --git a/CHANGES.md b/CHANGES.md index e4240eacfca..6278aed77d8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,8 @@ +- Consistently add trailing comma on typed parameters (#4164) + ### Configuration diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index 1cdd25fdb7c..d5faae36911 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -26,6 +26,8 @@ Currently, the following features are included in the preview style: brackets ([see below](labels/hug-parens)) - `no_normalize_fmt_skip_whitespace`: whitespace before `# fmt: skip` comments is no longer normalized +- `typed_params_trailing_comma`: consistently add trailing commas to typed function + parameters (labels/unstable-features)= diff --git a/src/black/files.py b/src/black/files.py index 65951efdbe8..1eb8745572b 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -131,7 +131,7 @@ def parse_pyproject_toml(path_config: str) -> Dict[str, Any]: def infer_target_version( - pyproject_toml: Dict[str, Any] + pyproject_toml: Dict[str, Any], ) -> Optional[List[TargetVersion]]: """Infer Black's target version from the project metadata in pyproject.toml. diff --git a/src/black/linegen.py b/src/black/linegen.py index a276805f2fe..c74ff9c0b4b 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -48,6 +48,7 @@ is_one_sequence_between, is_one_tuple, is_parent_function_or_class, + is_part_of_annotation, is_rpar_token, is_stub_body, is_stub_suite, @@ -1041,7 +1042,14 @@ def bracket_split_build_line( no_commas = ( original.is_def and opening_bracket.value == "(" - and not any(leaf.type == token.COMMA for leaf in leaves) + and not any( + leaf.type == token.COMMA + and ( + Preview.typed_params_trailing_comma not in original.mode + or not is_part_of_annotation(leaf) + ) + for leaf in leaves + ) # In particular, don't add one within a parenthesized return annotation. # Unfortunately the indicator we're in a return annotation (RARROW) may # be defined directly in the parent node, the parent of the parent ... diff --git a/src/black/mode.py b/src/black/mode.py index 128d2b9f108..22352e7c6a8 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -176,6 +176,7 @@ class Preview(Enum): no_normalize_fmt_skip_whitespace = auto() wrap_long_dict_values_in_parens = auto() multiline_string_handling = auto() + typed_params_trailing_comma = auto() UNSTABLE_FEATURES: Set[Preview] = { diff --git a/src/blib2to3/pgen2/parse.py b/src/blib2to3/pgen2/parse.py index ad51a3dad08..ad1d795b51a 100644 --- a/src/blib2to3/pgen2/parse.py +++ b/src/blib2to3/pgen2/parse.py @@ -50,7 +50,7 @@ def lam_sub(grammar: Grammar, node: RawNode) -> NL: def stack_copy( - stack: List[Tuple[DFAS, int, RawNode]] + stack: List[Tuple[DFAS, int, RawNode]], ) -> List[Tuple[DFAS, int, RawNode]]: """Nodeless stack copy.""" return [(dfa, label, DUMMY_NODE) for dfa, label, _ in stack] diff --git a/tests/data/cases/typed_params_trailing_comma.py b/tests/data/cases/typed_params_trailing_comma.py new file mode 100644 index 00000000000..a53b908b18b --- /dev/null +++ b/tests/data/cases/typed_params_trailing_comma.py @@ -0,0 +1,24 @@ +# flags: --preview +def long_function_name_goes_here( + x: Callable[List[int]] +) -> Union[List[int], float, str, bytes, Tuple[int]]: + pass + + +def long_function_name_goes_here( + x: Callable[[str, Any], int] +) -> Union[List[int], float, str, bytes, Tuple[int]]: + pass + + +# output +def long_function_name_goes_here( + x: Callable[List[int]], +) -> Union[List[int], float, str, bytes, Tuple[int]]: + pass + + +def long_function_name_goes_here( + x: Callable[[str, Any], int], +) -> Union[List[int], float, str, bytes, Tuple[int]]: + pass From 79fc1158a98281dac798feb14b8fddb4051e4a42 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 27 Jan 2024 23:24:36 -0500 Subject: [PATCH 32/68] chore: ignore node_modules (produced by a pre-commit check) (#4184) Signed-off-by: Henry Schreiner --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4a4f1b738ad..6e23797d110 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ src/_black_version.py .hypothesis/ venv/ .ipynb_checkpoints/ +node_modules/ From e026c93888f91a47a9c9f4e029f3eb07d96375e6 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 27 Jan 2024 20:51:32 -0800 Subject: [PATCH 33/68] Prepare release 24.1.1 (#4186) --- CHANGES.md | 44 ++------------------- docs/integrations/source_version_control.md | 4 +- docs/usage_and_configuration/the_basics.md | 6 +-- 3 files changed, 8 insertions(+), 46 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6278aed77d8..a794f421694 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,57 +1,19 @@ # Change Log -## Unreleased +## 24.1.1 -### Highlights - - - -### Stable style - - +Bugfix release to fix a bug that made Black unusable on certain file systems with strict +limits on path length. ### Preview style - - - Consistently add trailing comma on typed parameters (#4164) ### Configuration - - - Shorten the length of the name of the cache file to fix crashes on file systems that do not support long paths (#4176) -### Packaging - - - -### Parser - - - -### Performance - - - -### Output - - - -### _Blackd_ - - - -### Integrations - - - -### Documentation - - - ## 24.1.0 ### Highlights diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index 259c1c1eaf3..92279707d84 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -8,7 +8,7 @@ Use [pre-commit](https://pre-commit.com/). Once you repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.1.0 + rev: 24.1.1 hooks: - id: black # It is recommended to specify the latest version of Python @@ -35,7 +35,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.1.0 + rev: 24.1.1 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 562fd7d5905..dc9d9a64c68 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -266,8 +266,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 24.1.0 (compiled: yes) -$ black --required-version 24.1.0 -c "format = 'this'" +black, 24.1.1 (compiled: yes) +$ black --required-version 24.1.1 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -363,7 +363,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 24.1.0 +black, 24.1.1 ``` #### `--config` From 0b4364b7e38356428565fceb706a76ebf9a373e5 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 28 Jan 2024 05:37:12 -0800 Subject: [PATCH 34/68] Add new release template --- CHANGES.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index a794f421694..5ee1394282e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,52 @@ # Change Log +## Unreleased + +### Highlights + + + +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + + ## 24.1.1 Bugfix release to fix a bug that made Black unusable on certain file systems with strict From d919746fae2ba47442fc33699a5569fa04db305d Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 28 Jan 2024 07:05:56 -0800 Subject: [PATCH 35/68] Swallow warnings when performing AST checks (#4189) Fixes #4188 --- CHANGES.md | 3 +++ src/black/parsing.py | 16 ++++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5ee1394282e..6da30afc707 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -34,6 +34,9 @@ +- Black will swallow any `SyntaxWarning`s or `DeprecationWarning`s produced by the `ast` + module when performing equivalence checks (#4189) + ### _Blackd_ diff --git a/src/black/parsing.py b/src/black/parsing.py index 178a7ef10e2..63c5e71a0fe 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -4,6 +4,7 @@ import ast import sys +import warnings from typing import Iterable, Iterator, List, Set, Tuple from black.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature @@ -109,13 +110,16 @@ def lib2to3_unparse(node: Node) -> str: return code -def parse_single_version( +def _parse_single_version( src: str, version: Tuple[int, int], *, type_comments: bool ) -> ast.AST: filename = "" - return ast.parse( - src, filename, feature_version=version, type_comments=type_comments - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", SyntaxWarning) + warnings.simplefilter("ignore", DeprecationWarning) + return ast.parse( + src, filename, feature_version=version, type_comments=type_comments + ) def parse_ast(src: str) -> ast.AST: @@ -125,7 +129,7 @@ def parse_ast(src: str) -> ast.AST: first_error = "" for version in sorted(versions, reverse=True): try: - return parse_single_version(src, version, type_comments=True) + return _parse_single_version(src, version, type_comments=True) except SyntaxError as e: if not first_error: first_error = str(e) @@ -133,7 +137,7 @@ def parse_ast(src: str) -> ast.AST: # Try to parse without type comments for version in sorted(versions, reverse=True): try: - return parse_single_version(src, version, type_comments=False) + return _parse_single_version(src, version, type_comments=False) except SyntaxError: pass From 177e306363b9da5fcef621d15c205453ad1cfc74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 28 Jan 2024 23:07:34 -0800 Subject: [PATCH 36/68] Bump pypa/cibuildwheel from 2.16.2 to 2.16.4 (#4191) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.16.2 to 2.16.4. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.16.2...v2.16.4) --- .github/workflows/pypi_upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 8e3eb67a10d..52525419f0a 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -89,7 +89,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: pypa/cibuildwheel@v2.16.2 + - uses: pypa/cibuildwheel@v2.16.4 with: only: ${{ matrix.only }} From 2bc5ce8ae1f1ad52909b4656444c1e9988e23cab Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 29 Jan 2024 10:56:48 -0500 Subject: [PATCH 37/68] feat: add schema and validate-pyproject support (#4181) Signed-off-by: Henry Schreiner Co-authored-by: Jelle Zijlstra --- .github/workflows/lint.yml | 5 + .github/workflows/test.yml | 2 +- .pre-commit-config.yaml | 11 +- CHANGES.md | 2 + pyproject.toml | 3 + scripts/generate_schema.py | 74 +++++++++++++ src/black/resources/__init__.py | 0 src/black/resources/black.schema.json | 149 ++++++++++++++++++++++++++ src/black/schema.py | 20 ++++ tests/test_schema.py | 17 +++ tox.ini | 10 +- 11 files changed, 288 insertions(+), 5 deletions(-) create mode 100644 scripts/generate_schema.py create mode 100644 src/black/resources/__init__.py create mode 100644 src/black/resources/black.schema.json create mode 100644 src/black/schema.py create mode 100644 tests/test_schema.py diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2d016cef7a6..8cb335bffb5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -40,3 +40,8 @@ jobs: - name: Format ourselves run: | tox -e run_self + + - name: Regenerate schema + run: | + tox -e generate_schema + git diff --exit-code diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 55359a23303..fc274208258 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -106,4 +106,4 @@ jobs: python -m pip install -e ".[uvloop]" - name: Format ourselves - run: python -m black --check . + run: python -m black --check src/ tests/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 13479565527..05569eed8db 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,9 +42,9 @@ repos: rev: v1.8.0 hooks: - id: mypy - exclude: ^docs/conf.py - args: ["--config-file", "pyproject.toml"] - additional_dependencies: + exclude: ^(docs/conf.py|scripts/generate_schema.py)$ + args: [] + additional_dependencies: &mypy_deps - types-PyYAML - tomli >= 0.2.6, < 2.0.0 - click >= 8.1.0, != 8.1.4, != 8.1.5 @@ -56,6 +56,11 @@ repos: - types-commonmark - urllib3 - hypothesmith + - id: mypy + name: mypy (Python 3.10) + files: scripts/generate_schema.py + args: ["--python-version=3.10"] + additional_dependencies: *mypy_deps - repo: https://github.com/pre-commit/mirrors-prettier rev: v4.0.0-alpha.8 diff --git a/CHANGES.md b/CHANGES.md index 6da30afc707..4fd030df6df 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -45,6 +45,8 @@ +- Add a JSONSchema and provide a validate-pyproject entry-point (#4181) + ### Documentation +- _Black_ now ignores `pyproject.toml` that is missing a `tool.black` section when + discovering project root and configuration. Since _Black_ continues to use version + control as an indicator of project root, this is expected to primarily change behavior + for users in a monorepo setup (desirably). If you wish to preserve previous behavior, + simply add an empty `[tool.black]` to the previously discovered `pyproject.toml` + (#4204) ### Packaging diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index dc9d9a64c68..61c52450165 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -456,10 +456,11 @@ of tools like [Poetry](https://python-poetry.org/), ### Where _Black_ looks for the file -By default _Black_ looks for `pyproject.toml` starting from the common base directory of -all files and directories passed on the command line. If it's not there, it looks in -parent directories. It stops looking when it finds the file, or a `.git` directory, or a -`.hg` directory, or the root of the file system, whichever comes first. +By default _Black_ looks for `pyproject.toml` containing a `[tool.black]` section +starting from the common base directory of all files and directories passed on the +command line. If it's not there, it looks in parent directories. It stops looking when +it finds the file, or a `.git` directory, or a `.hg` directory, or the root of the file +system, whichever comes first. If you're formatting standard input, _Black_ will look for configuration starting from the current working directory. diff --git a/src/black/files.py b/src/black/files.py index 1eb8745572b..960f13ee270 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -42,6 +42,12 @@ import colorama # noqa: F401 +@lru_cache +def _load_toml(path: Union[Path, str]) -> Dict[str, Any]: + with open(path, "rb") as f: + return tomllib.load(f) + + @lru_cache def find_project_root( srcs: Sequence[str], stdin_filename: Optional[str] = None @@ -84,7 +90,9 @@ def find_project_root( return directory, ".hg directory" if (directory / "pyproject.toml").is_file(): - return directory, "pyproject.toml" + pyproject_toml = _load_toml(directory / "pyproject.toml") + if "black" in pyproject_toml.get("tool", {}): + return directory, "pyproject.toml" return directory, "file system root" @@ -117,8 +125,7 @@ def parse_pyproject_toml(path_config: str) -> Dict[str, Any]: If parsing fails, will raise a tomllib.TOMLDecodeError. """ - with open(path_config, "rb") as f: - pyproject_toml = tomllib.load(f) + pyproject_toml = _load_toml(path_config) config: Dict[str, Any] = pyproject_toml.get("tool", {}).get("black", {}) config = {k.replace("--", "").replace("-", "_"): v for k, v in config.items()} diff --git a/tests/test_black.py b/tests/test_black.py index 123ea0bb88a..f876d365b12 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1668,9 +1668,9 @@ def test_find_project_root(self) -> None: src_dir.mkdir() root_pyproject = root / "pyproject.toml" - root_pyproject.touch() + root_pyproject.write_text("[tool.black]", encoding="utf-8") src_pyproject = src_dir / "pyproject.toml" - src_pyproject.touch() + src_pyproject.write_text("[tool.black]", encoding="utf-8") src_python = src_dir / "foo.py" src_python.touch() @@ -1693,6 +1693,20 @@ def test_find_project_root(self) -> None: (src_dir.resolve(), "pyproject.toml"), ) + src_sub = src_dir / "sub" + src_sub.mkdir() + + src_sub_pyproject = src_sub / "pyproject.toml" + src_sub_pyproject.touch() # empty + + src_sub_python = src_sub / "bar.py" + + # we skip src_sub_pyproject since it is missing the [tool.black] section + self.assertEqual( + black.find_project_root((src_sub_python,)), + (src_dir.resolve(), "pyproject.toml"), + ) + @patch( "black.files.find_user_pyproject_toml", ) From 9728b8e9b8720b5e401249c077e32de46a479161 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 1 Feb 2024 21:58:51 -0800 Subject: [PATCH 43/68] Move hug_parens_with_braces_and_square_brackets into the unstable style (#4198) Primarily because of #4036 (a crash) but also because of the feedback in #4098 and #4099. --- CHANGES.md | 3 +++ docs/the_black_code_style/future_style.md | 4 ++-- src/black/mode.py | 2 ++ .../preview_hug_parens_with_braces_and_square_brackets.py | 2 +- ...eview_hug_parens_with_braces_and_square_brackets_no_ll1.py | 2 +- 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a3ffba610d9..4d646a2779e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,9 @@ +- Move the `hug_parens_with_braces_and_square_brackets` feature to the unstable style + due to an outstanding crash and proposed formatting tweaks (#4198) + ### Configuration - _Black_ now ignores `pyproject.toml` that is missing a `tool.black` section when diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index 1c11186f811..86e5aa806b2 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -24,8 +24,6 @@ Currently, the following features are included in the preview style: strings - `unify_docstring_detection`: fix inconsistencies in whether certain strings are detected as docstrings -- `hug_parens_with_braces_and_square_brackets`: more compact formatting of nested - brackets ([see below](labels/hug-parens)) - `no_normalize_fmt_skip_whitespace`: whitespace before `# fmt: skip` comments is no longer normalized - `typed_params_trailing_comma`: consistently add trailing commas to typed function @@ -41,6 +39,8 @@ The unstable style additionally includes the following features: ([see below](labels/wrap-long-dict-values)) - `multiline_string_handling`: more compact formatting of expressions involving multiline strings ([see below](labels/multiline-string-handling)) +- `hug_parens_with_braces_and_square_brackets`: more compact formatting of nested + brackets ([see below](labels/hug-parens)) (labels/hug-parens)= diff --git a/src/black/mode.py b/src/black/mode.py index 22352e7c6a8..5738bd6b793 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -186,6 +186,8 @@ class Preview(Enum): Preview.wrap_long_dict_values_in_parens, # See issue #4159 Preview.multiline_string_handling, + # See issue #4036 (crash), #4098, #4099 (proposed tweaks) + Preview.hug_parens_with_braces_and_square_brackets, } diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py index 47a6a0bcae6..cbbcf16d3bd 100644 --- a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable def foo_brackets(request): return JsonResponse( { diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets_no_ll1.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets_no_ll1.py index fdebdf69c20..16ebea379bc 100644 --- a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets_no_ll1.py +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets_no_ll1.py @@ -1,4 +1,4 @@ -# flags: --preview --no-preview-line-length-1 +# flags: --unstable --no-preview-line-length-1 # split out from preview_hug_parens_with_brackes_and_square_brackets, as it produces # different code on the second pass with line-length 1 in many cases. # Seems to be about whether the last string in a sequence gets wrapped in parens or not. From 632f44bd68b818cbc9dfd57e7485f0e5c3863b76 Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Fri, 2 Feb 2024 00:00:41 -0600 Subject: [PATCH 44/68] docs: Refactor pycodestyle/Flake8 compatibility docs (#4194) Signed-off-by: RedGuy12 <61329810+RedGuy12@users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Co-authored-by: Jelle Zijlstra --- .flake8 | 4 +- docs/compatible_configs/flake8/.flake8 | 2 +- docs/compatible_configs/flake8/setup.cfg | 2 +- docs/compatible_configs/flake8/tox.ini | 2 +- docs/compatible_configs/pycodestyle/.flake8 | 3 + docs/compatible_configs/pycodestyle/setup.cfg | 3 + docs/compatible_configs/pycodestyle/tox.ini | 3 + docs/faq.md | 9 +- docs/guides/using_black_with_other_tools.md | 130 ++++++++++-------- docs/the_black_code_style/current_style.md | 33 +---- 10 files changed, 94 insertions(+), 97 deletions(-) create mode 100644 docs/compatible_configs/pycodestyle/.flake8 create mode 100644 docs/compatible_configs/pycodestyle/setup.cfg create mode 100644 docs/compatible_configs/pycodestyle/tox.ini diff --git a/.flake8 b/.flake8 index 85f51cf9f05..f8dca18e7cf 100644 --- a/.flake8 +++ b/.flake8 @@ -1,8 +1,8 @@ [flake8] # B905 should be enabled when we drop support for 3.9 -ignore = E203, E266, E501, E704, W503, B905, B907 +ignore = E203, E266, E501, E701, E704, W503, B905, B907 # line length is intentionally set to 80 here because black uses Bugbear -# See https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#line-length for more details +# See https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#bugbear for more details max-line-length = 80 max-complexity = 18 select = B,C,E,F,W,T4,B9 diff --git a/docs/compatible_configs/flake8/.flake8 b/docs/compatible_configs/flake8/.flake8 index 8dd399ab55b..0d4ade348d6 100644 --- a/docs/compatible_configs/flake8/.flake8 +++ b/docs/compatible_configs/flake8/.flake8 @@ -1,3 +1,3 @@ [flake8] max-line-length = 88 -extend-ignore = E203 +extend-ignore = E203,E701 diff --git a/docs/compatible_configs/flake8/setup.cfg b/docs/compatible_configs/flake8/setup.cfg index 8dd399ab55b..0d4ade348d6 100644 --- a/docs/compatible_configs/flake8/setup.cfg +++ b/docs/compatible_configs/flake8/setup.cfg @@ -1,3 +1,3 @@ [flake8] max-line-length = 88 -extend-ignore = E203 +extend-ignore = E203,E701 diff --git a/docs/compatible_configs/flake8/tox.ini b/docs/compatible_configs/flake8/tox.ini index 8dd399ab55b..0d4ade348d6 100644 --- a/docs/compatible_configs/flake8/tox.ini +++ b/docs/compatible_configs/flake8/tox.ini @@ -1,3 +1,3 @@ [flake8] max-line-length = 88 -extend-ignore = E203 +extend-ignore = E203,E701 diff --git a/docs/compatible_configs/pycodestyle/.flake8 b/docs/compatible_configs/pycodestyle/.flake8 new file mode 100644 index 00000000000..34225907524 --- /dev/null +++ b/docs/compatible_configs/pycodestyle/.flake8 @@ -0,0 +1,3 @@ +[pycodestyle] +max-line-length = 88 +ignore = E203,E701 diff --git a/docs/compatible_configs/pycodestyle/setup.cfg b/docs/compatible_configs/pycodestyle/setup.cfg new file mode 100644 index 00000000000..34225907524 --- /dev/null +++ b/docs/compatible_configs/pycodestyle/setup.cfg @@ -0,0 +1,3 @@ +[pycodestyle] +max-line-length = 88 +ignore = E203,E701 diff --git a/docs/compatible_configs/pycodestyle/tox.ini b/docs/compatible_configs/pycodestyle/tox.ini new file mode 100644 index 00000000000..34225907524 --- /dev/null +++ b/docs/compatible_configs/pycodestyle/tox.ini @@ -0,0 +1,3 @@ +[pycodestyle] +max-line-length = 88 +ignore = E203,E701 diff --git a/docs/faq.md b/docs/faq.md index 124a096efac..d19ff8e7ace 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -77,13 +77,10 @@ following will not be formatted: - invalid syntax, as it can't be safely distinguished from automagics in the absence of a running `IPython` kernel. -## Why are Flake8's E203 and W503 violated? +## Why does Flake8 report warnings? -Because they go against PEP 8. E203 falsely triggers on list -[slices](the_black_code_style/current_style.md#slices), and adhering to W503 hinders -readability because operators are misaligned. Disable W503 and enable the -disabled-by-default counterpart W504. E203 should be disabled while changes are still -[discussed](https://github.com/PyCQA/pycodestyle/issues/373). +Some of Flake8's rules conflict with Black's style. We recommend disabling these rules. +See [Using _Black_ with other tools](labels/why-pycodestyle-warnings). ## Which Python versions does Black support? diff --git a/docs/guides/using_black_with_other_tools.md b/docs/guides/using_black_with_other_tools.md index e642a1aef33..187e3a3e6f5 100644 --- a/docs/guides/using_black_with_other_tools.md +++ b/docs/guides/using_black_with_other_tools.md @@ -134,10 +134,10 @@ profile = black -### Flake8 +### pycodestyle -[Flake8](https://pypi.org/p/flake8/) is a code linter. It warns you of syntax errors, -possible bugs, stylistic errors, etc. For the most part, Flake8 follows +[pycodestyle](https://pycodestyle.pycqa.org/) is a code linter. It warns you of syntax +errors, possible bugs, stylistic errors, etc. For the most part, pycodestyle follows [PEP 8](https://www.python.org/dev/peps/pep-0008/) when warning about stylistic errors. There are a few deviations that cause incompatibilities with _Black_. @@ -145,67 +145,115 @@ There are a few deviations that cause incompatibilities with _Black_. ``` max-line-length = 88 -extend-ignore = E203, E704 +ignore = E203,E701 ``` +(labels/why-pycodestyle-warnings)= + #### Why those options above? +##### `max-line-length` + +As with isort, pycodestyle should be configured to allow lines up to the length limit of +`88`, _Black_'s default. + +##### `E203` + In some cases, as determined by PEP 8, _Black_ will enforce an equal amount of -whitespace around slice operators. Due to this, Flake8 will raise -`E203 whitespace before ':'` warnings. Since this warning is not PEP 8 compliant, Flake8 -should be configured to ignore it via `extend-ignore = E203`. +whitespace around slice operators. Due to this, pycodestyle will raise +`E203 whitespace before ':'` warnings. Since this warning is not PEP 8 compliant, it +should be disabled. + +##### `E701` / `E704` + +_Black_ will collapse implementations of classes and functions consisting solely of `..` +to a single line. This matches how such examples are formatted in PEP 8. It remains true +that in all other cases Black will prevent multiple statements on the same line, in +accordance with PEP 8 generally discouraging this. + +However, `pycodestyle` does not mirror this logic and may raise +`E701 multiple statements on one line (colon)` in this situation. Its +disabled-by-default `E704 multiple statements on one line (def)` rule may also raise +warnings and should not be enabled. + +##### `W503` When breaking a line, _Black_ will break it before a binary operator. This is compliant with PEP 8 as of [April 2016](https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b#diff-64ec08cc46db7540f18f2af46037f599). There's a disabled-by-default warning in Flake8 which goes against this PEP 8 recommendation called `W503 line break before binary operator`. It should not be enabled -in your configuration. - -Also, as like with isort, flake8 should be configured to allow lines up to the length -limit of `88`, _Black_'s default. This explains `max-line-length = 88`. +in your configuration. You can use its counterpart +`W504 line break after binary operator` instead. #### Formats
-.flake8 +setup.cfg, .pycodestyle, tox.ini ```ini -[flake8] +[pycodestyle] max-line-length = 88 -extend-ignore = E203, E704 +ignore = E203,E701 ```
-
-setup.cfg +### Flake8 -```ini +[Flake8](https://pypi.org/p/flake8/) is a wrapper around multiple linters, including +pycodestyle. As such, it has many of the same issues. + +#### Bugbear + +It's recommended to use [the Bugbear plugin](https://github.com/PyCQA/flake8-bugbear) +and enable +[its B950 check](https://github.com/PyCQA/flake8-bugbear#opinionated-warnings#:~:text=you%20expect%20it.-,B950,-%3A%20Line%20too%20long) +instead of using Flake8's E501, because it aligns with +[Black's 10% rule](labels/line-length). + +Install Bugbear and use the following config: + +``` +[flake8] +max-line-length = 80 +extend-select = B950 +extend-ignore = E203,E501,E701 +``` + +#### Minimal Configuration + +In cases where you can't or don't want to install Bugbear, you can use this minimally +compatible config: + +``` [flake8] max-line-length = 88 -extend-ignore = E203, E704 +extend-ignore = E203,E701 ``` -
+#### Why those options above? + +See [the pycodestyle section](labels/why-pycodestyle-warnings) above. + +#### Formats
-tox.ini +.flake8, setup.cfg, tox.ini ```ini [flake8] max-line-length = 88 -extend-ignore = E203, E704 +extend-ignore = E203,E701 ```
### Pylint -[Pylint](https://pypi.org/p/pylint/) is also a code linter like Flake8. It has the same -checks as flake8 and more. In particular, it has more formatting checks regarding style -conventions like variable naming. With so many checks, Pylint is bound to have some -mixed feelings about _Black_'s formatting style. +[Pylint](https://pypi.org/p/pylint/) is also a code linter like Flake8. It has many of +the same checks as Flake8 and more. It particularly has more formatting checks regarding +style conventions like variable naming. #### Configuration @@ -252,35 +300,3 @@ max-line-length = "88" ``` - -### pycodestyle - -[pycodestyle](https://pycodestyle.pycqa.org/) is also a code linter like Flake8. - -#### Configuration - -``` -max-line-length = 88 -ignore = E203 -``` - -#### Why those options above? - -pycodestyle should be configured to only complain about lines that surpass `88` -characters via `max_line_length = 88`. - -See -[Why are Flake8’s E203 and W503 violated?](https://black.readthedocs.io/en/stable/faq.html#why-are-flake8-s-e203-and-w503-violated) - -#### Formats - -
-setup.cfg - -```cfg -[pycodestyle] -ignore = E203 -max_line_length = 88 -``` - -
diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index ca5d1d4a701..586c79074af 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -143,7 +143,7 @@ significantly shorter files than sticking with 80 (the most popular), or even 79 by the standard library). In general, [90-ish seems like the wise choice](https://youtu.be/wf-BqAjZb8M?t=260). -If you're paid by the line of code you write, you can pass `--line-length` with a lower +If you're paid by the lines of code you write, you can pass `--line-length` with a lower number. _Black_ will try to respect that. However, sometimes it won't be able to without breaking other rules. In those rare cases, auto-formatted code will exceed your allotted limit. @@ -153,35 +153,10 @@ harder to work with line lengths exceeding 100 characters. It also adversely aff side-by-side diff review on typical screen resolutions. Long lines also make it harder to present code neatly in documentation or talk slides. -#### Flake8 +#### Flake8 and other linters -If you use Flake8, you have a few options: - -1. Recommended is using [Bugbear](https://github.com/PyCQA/flake8-bugbear) and enabling - its B950 check instead of using Flake8's E501, because it aligns with Black's 10% - rule. Install Bugbear and use the following config: - - ```ini - [flake8] - max-line-length = 80 - ... - select = C,E,F,W,B,B950 - extend-ignore = E203, E501, E704 - ``` - - The rationale for B950 is explained in - [Bugbear's documentation](https://github.com/PyCQA/flake8-bugbear#opinionated-warnings). - -2. For a minimally compatible config: - - ```ini - [flake8] - max-line-length = 88 - extend-ignore = E203, E704 - ``` - -An explanation of why E203 is disabled can be found in the [Slices section](#slices) of -this page. +See [Using _Black_ with other tools](../guides/using_black_with_other_tools.md) about +linter compatibility. ### Empty lines From a08b480a2f39fb4fc7b210d8b450e33d3879f77d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 01:09:21 -0800 Subject: [PATCH 45/68] Bump pypa/cibuildwheel from 2.16.4 to 2.16.5 (#4212) --- .github/workflows/pypi_upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 52525419f0a..3c99f66cc7b 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -89,7 +89,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: pypa/cibuildwheel@v2.16.4 + - uses: pypa/cibuildwheel@v2.16.5 with: only: ${{ matrix.only }} From 3e80de3447dee272e9977ab58b1560a669b7b848 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 04:47:35 -0800 Subject: [PATCH 46/68] Bump furo from 2023.9.10 to 2024.1.29 in /docs (#4211) Bumps [furo](https://github.com/pradyunsg/furo) from 2023.9.10 to 2024.1.29. - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2023.09.10...2024.01.29) --- updated-dependencies: - dependency-name: furo dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index b5b9e22fc84..3bc058a0721 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ Sphinx==7.2.6 docutils==0.20.1 sphinxcontrib-programoutput==0.17 sphinx_copybutton==0.5.2 -furo==2023.9.10 +furo==2024.1.29 From 7edb50f5a0afc56bb648dc14640ced144366b43a Mon Sep 17 00:00:00 2001 From: Brandon J <153339574+veryslowcode@users.noreply.github.com> Date: Mon, 5 Feb 2024 05:56:07 -0700 Subject: [PATCH 47/68] fix: additional newline added to docstring when the previous line length is less than the line length limit minus 1 (#4185) Co-authored-by: Jelle Zijlstra --- CHANGES.md | 2 ++ docs/the_black_code_style/future_style.md | 2 ++ src/black/linegen.py | 13 ++++++++++--- src/black/mode.py | 1 + src/black/resources/black.schema.json | 3 ++- tests/data/cases/docstring_newline_preview.py | 4 ++++ 6 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 tests/data/cases/docstring_newline_preview.py diff --git a/CHANGES.md b/CHANGES.md index 4d646a2779e..a726a91457a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,8 @@ - Move the `hug_parens_with_braces_and_square_brackets` feature to the unstable style due to an outstanding crash and proposed formatting tweaks (#4198) +- Checking for newline before adding one on docstring that is almost at the line limit + (#4185) ### Configuration diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index 86e5aa806b2..d7640765b30 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -28,6 +28,8 @@ Currently, the following features are included in the preview style: longer normalized - `typed_params_trailing_comma`: consistently add trailing commas to typed function parameters +- `docstring_check_for_newline`: checks if there is a newline before the terminating + quotes of a docstring (labels/unstable-features)= diff --git a/src/black/linegen.py b/src/black/linegen.py index c74ff9c0b4b..c45a1308013 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -477,15 +477,22 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: last_line_length = len(lines[-1]) if docstring else 0 # If adding closing quotes would cause the last line to exceed - # the maximum line length then put a line break before the - # closing quotes + # the maximum line length, and the closing quote is not + # prefixed by a newline then put a line break before + # the closing quotes if ( len(lines) > 1 and last_line_length + quote_len > self.mode.line_length and len(indent) + quote_len <= self.mode.line_length and not has_trailing_backslash ): - leaf.value = prefix + quote + docstring + "\n" + indent + quote + if ( + Preview.docstring_check_for_newline in self.mode + and leaf.value[-1 - quote_len] == "\n" + ): + leaf.value = prefix + quote + docstring + quote + else: + leaf.value = prefix + quote + docstring + "\n" + indent + quote else: leaf.value = prefix + quote + docstring + quote else: diff --git a/src/black/mode.py b/src/black/mode.py index 5738bd6b793..9593a90d170 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -177,6 +177,7 @@ class Preview(Enum): wrap_long_dict_values_in_parens = auto() multiline_string_handling = auto() typed_params_trailing_comma = auto() + docstring_check_for_newline = auto() UNSTABLE_FEATURES: Set[Preview] = { diff --git a/src/black/resources/black.schema.json b/src/black/resources/black.schema.json index 40bf39137f7..5b1b1320eb4 100644 --- a/src/black/resources/black.schema.json +++ b/src/black/resources/black.schema.json @@ -85,7 +85,8 @@ "no_normalize_fmt_skip_whitespace", "wrap_long_dict_values_in_parens", "multiline_string_handling", - "typed_params_trailing_comma" + "typed_params_trailing_comma", + "docstring_check_for_newline" ] }, "description": "Enable specific features included in the `--unstable` style. Requires `--preview`. No compatibility guarantees are provided on the behavior or existence of any unstable features." diff --git a/tests/data/cases/docstring_newline_preview.py b/tests/data/cases/docstring_newline_preview.py new file mode 100644 index 00000000000..5c129ca5f80 --- /dev/null +++ b/tests/data/cases/docstring_newline_preview.py @@ -0,0 +1,4 @@ +# flags: --preview +""" +87 characters ............................................................................ +""" From 32230e6f5c4bc6bb5c469451e15f3f54d9884b51 Mon Sep 17 00:00:00 2001 From: Seung Wan Yoo <74849806+wannieman98@users.noreply.github.com> Date: Mon, 5 Feb 2024 22:33:11 +0900 Subject: [PATCH 48/68] fix: bug where the doublestar operation had inconsistent formatting. (#4154) Co-authored-by: Jelle Zijlstra --- CHANGES.md | 2 + docs/the_black_code_style/future_style.md | 2 + src/black/mode.py | 1 + src/black/resources/black.schema.json | 1 + src/black/trans.py | 138 ++++++++++++++---- ...simple_lookup_for_doublestar_expression.py | 14 ++ 6 files changed, 132 insertions(+), 26 deletions(-) create mode 100644 tests/data/cases/is_simple_lookup_for_doublestar_expression.py diff --git a/CHANGES.md b/CHANGES.md index a726a91457a..5d4858da811 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,8 @@ - Move the `hug_parens_with_braces_and_square_brackets` feature to the unstable style due to an outstanding crash and proposed formatting tweaks (#4198) +- Fixed a bug where base expressions caused inconsistent formatting of \*\* in tenary + expression (#4154) - Checking for newline before adding one on docstring that is almost at the line limit (#4185) diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index d7640765b30..f4534680645 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -28,6 +28,8 @@ Currently, the following features are included in the preview style: longer normalized - `typed_params_trailing_comma`: consistently add trailing commas to typed function parameters +- `is_simple_lookup_for_doublestar_expression`: fix line length computation for certain + expressions that involve the power operator - `docstring_check_for_newline`: checks if there is a newline before the terminating quotes of a docstring diff --git a/src/black/mode.py b/src/black/mode.py index 9593a90d170..f9fad082e03 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -177,6 +177,7 @@ class Preview(Enum): wrap_long_dict_values_in_parens = auto() multiline_string_handling = auto() typed_params_trailing_comma = auto() + is_simple_lookup_for_doublestar_expression = auto() docstring_check_for_newline = auto() diff --git a/src/black/resources/black.schema.json b/src/black/resources/black.schema.json index 5b1b1320eb4..04ddb32e087 100644 --- a/src/black/resources/black.schema.json +++ b/src/black/resources/black.schema.json @@ -86,6 +86,7 @@ "wrap_long_dict_values_in_parens", "multiline_string_handling", "typed_params_trailing_comma", + "is_simple_lookup_for_doublestar_expression", "docstring_check_for_newline" ] }, diff --git a/src/black/trans.py b/src/black/trans.py index 7c7335a005b..29a978c6b71 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -29,7 +29,7 @@ from black.comments import contains_pragma_comment from black.lines import Line, append_leaves -from black.mode import Feature, Mode +from black.mode import Feature, Mode, Preview from black.nodes import ( CLOSING_BRACKETS, OPENING_BRACKETS, @@ -94,43 +94,36 @@ def hug_power_op( else: raise CannotTransform("No doublestar token was found in the line.") - def is_simple_lookup(index: int, step: Literal[1, -1]) -> bool: + def is_simple_lookup(index: int, kind: Literal[1, -1]) -> bool: # Brackets and parentheses indicate calls, subscripts, etc. ... # basically stuff that doesn't count as "simple". Only a NAME lookup # or dotted lookup (eg. NAME.NAME) is OK. - if step == -1: - disallowed = {token.RPAR, token.RSQB} - else: - disallowed = {token.LPAR, token.LSQB} - - while 0 <= index < len(line.leaves): - current = line.leaves[index] - if current.type in disallowed: - return False - if current.type not in {token.NAME, token.DOT} or current.value == "for": - # If the current token isn't disallowed, we'll assume this is simple as - # only the disallowed tokens are semantically attached to this lookup - # expression we're checking. Also, stop early if we hit the 'for' bit - # of a comprehension. - return True + if Preview.is_simple_lookup_for_doublestar_expression not in mode: + return original_is_simple_lookup_func(line, index, kind) - index += step - - return True + else: + if kind == -1: + return handle_is_simple_look_up_prev( + line, index, {token.RPAR, token.RSQB} + ) + else: + return handle_is_simple_lookup_forward( + line, index, {token.LPAR, token.LSQB} + ) - def is_simple_operand(index: int, kind: Literal["base", "exponent"]) -> bool: + def is_simple_operand(index: int, kind: Literal[1, -1]) -> bool: # An operand is considered "simple" if's a NAME, a numeric CONSTANT, a simple # lookup (see above), with or without a preceding unary operator. start = line.leaves[index] if start.type in {token.NAME, token.NUMBER}: - return is_simple_lookup(index, step=(1 if kind == "exponent" else -1)) + return is_simple_lookup(index, kind) if start.type in {token.PLUS, token.MINUS, token.TILDE}: if line.leaves[index + 1].type in {token.NAME, token.NUMBER}: - # step is always one as bases with a preceding unary op will be checked + # kind is always one as bases with a preceding unary op will be checked # for simplicity starting from the next token (so it'll hit the check # above). - return is_simple_lookup(index + 1, step=1) + return is_simple_lookup(index + 1, kind=1) return False @@ -145,9 +138,9 @@ def is_simple_operand(index: int, kind: Literal["base", "exponent"]) -> bool: should_hug = ( (0 < idx < len(line.leaves) - 1) and leaf.type == token.DOUBLESTAR - and is_simple_operand(idx - 1, kind="base") + and is_simple_operand(idx - 1, kind=-1) and line.leaves[idx - 1].value != "lambda" - and is_simple_operand(idx + 1, kind="exponent") + and is_simple_operand(idx + 1, kind=1) ) if should_hug: new_leaf.prefix = "" @@ -162,6 +155,99 @@ def is_simple_operand(index: int, kind: Literal["base", "exponent"]) -> bool: yield new_line +def original_is_simple_lookup_func( + line: Line, index: int, step: Literal[1, -1] +) -> bool: + if step == -1: + disallowed = {token.RPAR, token.RSQB} + else: + disallowed = {token.LPAR, token.LSQB} + + while 0 <= index < len(line.leaves): + current = line.leaves[index] + if current.type in disallowed: + return False + if current.type not in {token.NAME, token.DOT} or current.value == "for": + # If the current token isn't disallowed, we'll assume this is + # simple as only the disallowed tokens are semantically + # attached to this lookup expression we're checking. Also, + # stop early if we hit the 'for' bit of a comprehension. + return True + + index += step + + return True + + +def handle_is_simple_look_up_prev(line: Line, index: int, disallowed: Set[int]) -> bool: + """ + Handling the determination of is_simple_lookup for the lines prior to the doublestar + token. This is required because of the need to isolate the chained expression + to determine the bracket or parenthesis belong to the single expression. + """ + contains_disallowed = False + chain = [] + + while 0 <= index < len(line.leaves): + current = line.leaves[index] + chain.append(current) + if not contains_disallowed and current.type in disallowed: + contains_disallowed = True + if not is_expression_chained(chain): + return not contains_disallowed + + index -= 1 + + return True + + +def handle_is_simple_lookup_forward( + line: Line, index: int, disallowed: Set[int] +) -> bool: + """ + Handling decision is_simple_lookup for the lines behind the doublestar token. + This function is simplified to keep consistent with the prior logic and the forward + case are more straightforward and do not need to care about chained expressions. + """ + while 0 <= index < len(line.leaves): + current = line.leaves[index] + if current.type in disallowed: + return False + if current.type not in {token.NAME, token.DOT} or ( + current.type == token.NAME and current.value == "for" + ): + # If the current token isn't disallowed, we'll assume this is simple as + # only the disallowed tokens are semantically attached to this lookup + # expression we're checking. Also, stop early if we hit the 'for' bit + # of a comprehension. + return True + + index += 1 + + return True + + +def is_expression_chained(chained_leaves: List[Leaf]) -> bool: + """ + Function to determine if the variable is a chained call. + (e.g., foo.lookup, foo().lookup, (foo.lookup())) will be recognized as chained call) + """ + if len(chained_leaves) < 2: + return True + + current_leaf = chained_leaves[-1] + past_leaf = chained_leaves[-2] + + if past_leaf.type == token.NAME: + return current_leaf.type in {token.DOT} + elif past_leaf.type in {token.RPAR, token.RSQB}: + return current_leaf.type in {token.RSQB, token.RPAR} + elif past_leaf.type in {token.LPAR, token.LSQB}: + return current_leaf.type in {token.NAME, token.LPAR, token.LSQB} + else: + return False + + class StringTransformer(ABC): """ An implementation of the Transformer protocol that relies on its diff --git a/tests/data/cases/is_simple_lookup_for_doublestar_expression.py b/tests/data/cases/is_simple_lookup_for_doublestar_expression.py new file mode 100644 index 00000000000..a0d2e2ba842 --- /dev/null +++ b/tests/data/cases/is_simple_lookup_for_doublestar_expression.py @@ -0,0 +1,14 @@ +# flags: --preview +m2 = None if not isinstance(dist, Normal) else m** 2 + s * 2 +m3 = None if not isinstance(dist, Normal) else m ** 2 + s * 2 +m4 = None if not isinstance(dist, Normal) else m**2 + s * 2 +m5 = obj.method(another_obj.method()).attribute **2 +m6 = None if ... else m**2 + s**2 + + +# output +m2 = None if not isinstance(dist, Normal) else m**2 + s * 2 +m3 = None if not isinstance(dist, Normal) else m**2 + s * 2 +m4 = None if not isinstance(dist, Normal) else m**2 + s * 2 +m5 = obj.method(another_obj.method()).attribute ** 2 +m6 = None if ... else m**2 + s**2 \ No newline at end of file From dab37a6a1117d690d683121edc4d7c8fb8dd75a7 Mon Sep 17 00:00:00 2001 From: Logan Hunt <39638017+dosisod@users.noreply.github.com> Date: Wed, 7 Feb 2024 06:55:02 -0800 Subject: [PATCH 49/68] Remove redundant parentheses in `case` statement `if` guards (#4214) A follow up to #4024 but for `if` guards in `case` statements. I noticed this when #4024 was made stable, and noticed I had some code that had extra parens around the `if` guard. --- CHANGES.md | 1 + docs/the_black_code_style/future_style.md | 2 + src/black/linegen.py | 2 + src/black/mode.py | 1 + src/black/resources/black.schema.json | 3 +- .../remove_redundant_parens_in_case_guard.py | 114 ++++++++++++++++++ 6 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 tests/data/cases/remove_redundant_parens_in_case_guard.py diff --git a/CHANGES.md b/CHANGES.md index 5d4858da811..3583348f377 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,6 +20,7 @@ expression (#4154) - Checking for newline before adding one on docstring that is almost at the line limit (#4185) +- Remove redundant parentheses in `case` statement `if` guards (#4214). ### Configuration diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index f4534680645..4ae46cecded 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -32,6 +32,8 @@ Currently, the following features are included in the preview style: expressions that involve the power operator - `docstring_check_for_newline`: checks if there is a newline before the terminating quotes of a docstring +- `remove_redundant_guard_parens`: Removes redundant parentheses in `if` guards for + `case` blocks. (labels/unstable-features)= diff --git a/src/black/linegen.py b/src/black/linegen.py index c45a1308013..2b7fbc492cf 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -529,6 +529,8 @@ def __post_init__(self) -> None: # PEP 634 self.visit_match_stmt = self.visit_match_case self.visit_case_block = self.visit_match_case + if Preview.remove_redundant_guard_parens in self.mode: + self.visit_guard = partial(v, keywords=Ø, parens={"if"}) def _hugging_power_ops_line_to_string( diff --git a/src/black/mode.py b/src/black/mode.py index f9fad082e03..90c10c324a5 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -179,6 +179,7 @@ class Preview(Enum): typed_params_trailing_comma = auto() is_simple_lookup_for_doublestar_expression = auto() docstring_check_for_newline = auto() + remove_redundant_guard_parens = auto() UNSTABLE_FEATURES: Set[Preview] = { diff --git a/src/black/resources/black.schema.json b/src/black/resources/black.schema.json index 04ddb32e087..8252a6c4bd8 100644 --- a/src/black/resources/black.schema.json +++ b/src/black/resources/black.schema.json @@ -87,7 +87,8 @@ "multiline_string_handling", "typed_params_trailing_comma", "is_simple_lookup_for_doublestar_expression", - "docstring_check_for_newline" + "docstring_check_for_newline", + "remove_redundant_guard_parens" ] }, "description": "Enable specific features included in the `--unstable` style. Requires `--preview`. No compatibility guarantees are provided on the behavior or existence of any unstable features." diff --git a/tests/data/cases/remove_redundant_parens_in_case_guard.py b/tests/data/cases/remove_redundant_parens_in_case_guard.py new file mode 100644 index 00000000000..bec4a3c3fcd --- /dev/null +++ b/tests/data/cases/remove_redundant_parens_in_case_guard.py @@ -0,0 +1,114 @@ +# flags: --minimum-version=3.10 --preview --line-length=79 + +match 1: + case _ if (True): + pass + + +match 1: + case _ if ( + True + ): + pass + + +match 1: + case _ if ( + # this is a comment + True + ): + pass + + +match 1: + case _ if ( + True + # this is a comment + ): + pass + + +match 1: + case _ if ( + True # this is a comment + ): + pass + + +match 1: + case _ if ( # this is a comment + True + ): + pass + + +match 1: + case _ if ( + True + ): # this is a comment + pass + + +match 1: + case _ if (True): # comment over the line limit unless parens are removed x + pass + + +match 1: + case _ if (True): # comment over the line limit and parens should go to next line + pass + + +# output + +match 1: + case _ if True: + pass + + +match 1: + case _ if True: + pass + + +match 1: + case _ if ( + # this is a comment + True + ): + pass + + +match 1: + case _ if ( + True + # this is a comment + ): + pass + + +match 1: + case _ if True: # this is a comment + pass + + +match 1: + case _ if True: # this is a comment + pass + + +match 1: + case _ if True: # this is a comment + pass + + +match 1: + case _ if True: # comment over the line limit unless parens are removed x + pass + + +match 1: + case ( + _ + ) if True: # comment over the line limit and parens should go to next line + pass From a20100395cf6179a81289452efad1d8e72b19682 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 10 Feb 2024 23:55:01 -0800 Subject: [PATCH 50/68] Simplify check for symlinks that resolve outside root (#4221) This PR does not change any behaviour. There have been 1-2 issues about symlinks recently. Both over and under resolving can cause problems. This makes a case where we resolve more explicit and prevent a resolved path from leaking out via the return. --- src/black/__init__.py | 11 ++++------- src/black/files.py | 25 +++++++++++-------------- tests/test_black.py | 11 +++++++---- 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 8ab5b47f974..2d4c7f655ad 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -50,9 +50,9 @@ gen_python_files, get_gitignore, get_root_relative_path, - normalize_path_maybe_ignore, parse_pyproject_toml, path_is_excluded, + resolves_outside_root_or_cannot_stat, wrap_stream_for_windows, ) from black.handle_ipynb_magics import ( @@ -763,12 +763,9 @@ def get_sources( ) continue - normalized_path: Optional[str] = normalize_path_maybe_ignore( - path, root, report - ) - if normalized_path is None: + if resolves_outside_root_or_cannot_stat(path, root, report): if verbose: - out(f'Skipping invalid source: "{normalized_path}"', fg="red") + out(f'Skipping invalid source: "{path}"', fg="red") continue if is_stdin: @@ -780,7 +777,7 @@ def get_sources( continue if verbose: - out(f'Found input source: "{normalized_path}"', fg="blue") + out(f'Found input source: "{path}"', fg="blue") sources.add(path) elif path.is_dir(): path = root / (path.resolve().relative_to(root)) diff --git a/src/black/files.py b/src/black/files.py index 960f13ee270..6c05105450c 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -254,26 +254,24 @@ def get_gitignore(root: Path) -> PathSpec: raise -def normalize_path_maybe_ignore( +def resolves_outside_root_or_cannot_stat( path: Path, root: Path, report: Optional[Report] = None, -) -> Optional[str]: - """Normalize `path`. May return `None` if `path` was ignored. - - `report` is where "path ignored" output goes. +) -> bool: + """ + Returns whether the path is a symbolic link that points outside the + root directory. Also returns True if we failed to resolve the path. """ try: - abspath = path if path.is_absolute() else Path.cwd() / path - normalized_path = abspath.resolve() - root_relative_path = get_root_relative_path(normalized_path, root, report) - + if sys.version_info < (3, 8, 6): + path = path.absolute() # https://bugs.python.org/issue33660 + resolved_path = path.resolve() + return get_root_relative_path(resolved_path, root, report) is None except OSError as e: if report: report.path_ignored(path, f"cannot be read because {e}") - return None - - return root_relative_path + return True def get_root_relative_path( @@ -369,8 +367,7 @@ def gen_python_files( report.path_ignored(child, "matches the --force-exclude regular expression") continue - normalized_path = normalize_path_maybe_ignore(child, root, report) - if normalized_path is None: + if resolves_outside_root_or_cannot_stat(child, root, report): continue if child.is_dir(): diff --git a/tests/test_black.py b/tests/test_black.py index f876d365b12..5c6920c2b77 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1760,12 +1760,15 @@ def test_bpo_33660_workaround(self) -> None: return # https://bugs.python.org/issue33660 + # Can be removed when we drop support for Python 3.8.5 root = Path("/") with change_directory(root): path = Path("workspace") / "project" report = black.Report(verbose=True) - normalized_path = black.normalize_path_maybe_ignore(path, root, report) - self.assertEqual(normalized_path, "workspace/project") + resolves_outside = black.resolves_outside_root_or_cannot_stat( + path, root, report + ) + self.assertIs(resolves_outside, False) def test_normalize_path_ignore_windows_junctions_outside_of_root(self) -> None: if system() != "Windows": @@ -1778,13 +1781,13 @@ def test_normalize_path_ignore_windows_junctions_outside_of_root(self) -> None: os.system(f"mklink /J {junction_dir} {junction_target_outside_of_root}") report = black.Report(verbose=True) - normalized_path = black.normalize_path_maybe_ignore( + resolves_outside = black.resolves_outside_root_or_cannot_stat( junction_dir, root, report ) # Manually delete for Python < 3.8 os.system(f"rmdir {junction_dir}") - self.assertEqual(normalized_path, None) + self.assertIs(resolves_outside, True) def test_newline_comment_interaction(self) -> None: source = "class A:\\\r\n# type: ignore\n pass\n" From 23dfc5b2c3b0694a8c27e58e28439591976aaf94 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 12 Feb 2024 00:04:09 -0800 Subject: [PATCH 51/68] Fix ignoring input files for symlink reasons (#4222) This relates to #4015, #4161 and the behaviour of os.getcwd() Black is a big user of pathlib and as such loves doing `.resolve()`, since for a long time it was the only good way of getting an absolute path in pathlib. However, this has two problems: The first minor problem is performance, e.g. in #3751 I (safely) got rid of a bunch of `.resolve()` which made Black 40% faster on cached runs. The second more important problem is that always resolving symlinks results in unintuitive exclusion behaviour. For instance, a gitignored symlink should never alter formatting of your actual code. This kind of thing was reported by users a few times. In #3846, I improved the exclusion rule logic for symlinks in `gen_python_files` and everything was good. But `gen_python_files` isn't enough, there's also `get_sources`, which handles user specified paths directly (instead of files Black discovers). So in #4015, I made a very similar change to #3846 for `get_sources`, and this is where some problems began. The core issue was the line: ``` root_relative_path = path.absolute().relative_to(root).as_posix() ``` The first issue is that despite root being computed from user inputs, we call `.resolve()` while computing it (likely unecessarily). Which means that `path` may not actually be relative to `root`. So I started off this PR trying to fix that, when I ran into the second issue. Which is that `os.getcwd()` (as called by `os.path.abspath` or `Path.absolute` or `Path.cwd`) also often resolves symlinks! ``` >>> import os >>> os.environ.get("PWD") '/Users/shantanu/dev/black/symlink/bug' >>> os.getcwd() '/Users/shantanu/dev/black/actual/bug' ``` This also meant that the breakage often would not show up when input relative paths. This doesn't affect `gen_python_files` / #3846 because things are always absolute and known to be relative to `root`. Anyway, it looks like #4161 fixed the crash by just swallowing the error and ignoring the file. Instead, we should just try to compute the actual relative path. I think this PR should be quite safe, but we could also consider reverting some of the previous changes; the associated issues weren't too popular. At the same time, I think there's still behaviour that can be improved and I kind of want to make larger changes, but maybe I'll save that for if we do something like #3952 Hopefully fixes #4205, fixes #4209, actual fix for #4077 --- CHANGES.md | 1 + src/black/__init__.py | 15 +++--- src/black/files.py | 44 ++++++++++------ tests/test_black.py | 120 ++++++++++++++++++++++++++++++++---------- 4 files changed, 127 insertions(+), 53 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3583348f377..4fd63ac41eb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -24,6 +24,7 @@ ### Configuration +- Fix issue where _Black_ would ignore input files in the presence of symlinks (#4222) - _Black_ now ignores `pyproject.toml` that is missing a `tool.black` section when discovering project root and configuration. Since _Black_ continues to use version control as an indicator of project root, this is expected to primarily change behavior diff --git a/src/black/__init__.py b/src/black/__init__.py index 2d4c7f655ad..f82b9fec5b7 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -44,12 +44,12 @@ STDIN_PLACEHOLDER, ) from black.files import ( + best_effort_relative_path, find_project_root, find_pyproject_toml, find_user_pyproject_toml, gen_python_files, get_gitignore, - get_root_relative_path, parse_pyproject_toml, path_is_excluded, resolves_outside_root_or_cannot_stat, @@ -734,6 +734,7 @@ def get_sources( """Compute the set of files to be formatted.""" sources: Set[Path] = set() + assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}" using_default_exclude = exclude is None exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES) if exclude is None else exclude gitignore: Optional[Dict[Path, PathSpec]] = None @@ -749,11 +750,12 @@ def get_sources( # Compare the logic here to the logic in `gen_python_files`. if is_stdin or path.is_file(): - root_relative_path = get_root_relative_path(path, root, report) - - if root_relative_path is None: + if resolves_outside_root_or_cannot_stat(path, root, report): + if verbose: + out(f'Skipping invalid source: "{path}"', fg="red") continue + root_relative_path = best_effort_relative_path(path, root).as_posix() root_relative_path = "/" + root_relative_path # Hard-exclude any files that matches the `--force-exclude` regex. @@ -763,11 +765,6 @@ def get_sources( ) continue - if resolves_outside_root_or_cannot_stat(path, root, report): - if verbose: - out(f'Skipping invalid source: "{path}"', fg="red") - continue - if is_stdin: path = Path(f"{STDIN_PLACEHOLDER}{str(path)}") diff --git a/src/black/files.py b/src/black/files.py index 6c05105450c..c0cadbfd890 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -48,6 +48,11 @@ def _load_toml(path: Union[Path, str]) -> Dict[str, Any]: return tomllib.load(f) +@lru_cache +def _cached_resolve(path: Path) -> Path: + return path.resolve() + + @lru_cache def find_project_root( srcs: Sequence[str], stdin_filename: Optional[str] = None @@ -67,9 +72,9 @@ def find_project_root( if stdin_filename is not None: srcs = tuple(stdin_filename if s == "-" else s for s in srcs) if not srcs: - srcs = [str(Path.cwd().resolve())] + srcs = [str(_cached_resolve(Path.cwd()))] - path_srcs = [Path(Path.cwd(), src).resolve() for src in srcs] + path_srcs = [_cached_resolve(Path(Path.cwd(), src)) for src in srcs] # A list of lists of parents for each 'src'. 'src' is included as a # "parent" of itself if it is a directory @@ -236,7 +241,7 @@ def find_user_pyproject_toml() -> Path: else: config_root = os.environ.get("XDG_CONFIG_HOME", "~/.config") user_config_path = Path(config_root).expanduser() / "black" - return user_config_path.resolve() + return _cached_resolve(user_config_path) @lru_cache @@ -266,27 +271,31 @@ def resolves_outside_root_or_cannot_stat( try: if sys.version_info < (3, 8, 6): path = path.absolute() # https://bugs.python.org/issue33660 - resolved_path = path.resolve() - return get_root_relative_path(resolved_path, root, report) is None + resolved_path = _cached_resolve(path) except OSError as e: if report: report.path_ignored(path, f"cannot be read because {e}") return True - - -def get_root_relative_path( - path: Path, - root: Path, - report: Optional[Report] = None, -) -> Optional[str]: - """Returns the file path relative to the 'root' directory""" try: - root_relative_path = path.absolute().relative_to(root).as_posix() + resolved_path.relative_to(root) except ValueError: if report: report.path_ignored(path, f"is a symbolic link that points outside {root}") - return None - return root_relative_path + return True + return False + + +def best_effort_relative_path(path: Path, root: Path) -> Path: + # Precondition: resolves_outside_root_or_cannot_stat(path, root) is False + try: + return path.absolute().relative_to(root) + except ValueError: + pass + root_parent = next((p for p in path.parents if _cached_resolve(p) == root), None) + if root_parent is not None: + return path.relative_to(root_parent) + # something adversarial, fallback to path guaranteed by precondition + return _cached_resolve(path).relative_to(root) def _path_is_ignored( @@ -339,7 +348,8 @@ def gen_python_files( assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}" for child in paths: - root_relative_path = child.absolute().relative_to(root).as_posix() + assert child.is_absolute() + root_relative_path = child.relative_to(root).as_posix() # First ignore files matching .gitignore, if passed if gitignore_dict and _path_is_ignored( diff --git a/tests/test_black.py b/tests/test_black.py index 5c6920c2b77..41f87cd16f8 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2567,32 +2567,32 @@ def test_symlinks(self) -> None: gitignore = PathSpec.from_lines("gitwildmatch", []) regular = MagicMock() - regular.absolute.return_value = root / "regular.py" + regular.relative_to.return_value = Path("regular.py") regular.resolve.return_value = root / "regular.py" regular.is_dir.return_value = False regular.is_file.return_value = True outside_root_symlink = MagicMock() - outside_root_symlink.absolute.return_value = root / "symlink.py" + outside_root_symlink.relative_to.return_value = Path("symlink.py") outside_root_symlink.resolve.return_value = Path("/nowhere") outside_root_symlink.is_dir.return_value = False outside_root_symlink.is_file.return_value = True ignored_symlink = MagicMock() - ignored_symlink.absolute.return_value = root / ".mypy_cache" / "symlink.py" + ignored_symlink.relative_to.return_value = Path(".mypy_cache") / "symlink.py" ignored_symlink.is_dir.return_value = False ignored_symlink.is_file.return_value = True # A symlink that has an excluded name, but points to an included name symlink_excluded_name = MagicMock() - symlink_excluded_name.absolute.return_value = root / "excluded_name" + symlink_excluded_name.relative_to.return_value = Path("excluded_name") symlink_excluded_name.resolve.return_value = root / "included_name.py" symlink_excluded_name.is_dir.return_value = False symlink_excluded_name.is_file.return_value = True # A symlink that has an included name, but points to an excluded name symlink_included_name = MagicMock() - symlink_included_name.absolute.return_value = root / "included_name.py" + symlink_included_name.relative_to.return_value = Path("included_name.py") symlink_included_name.resolve.return_value = root / "excluded_name" symlink_included_name.is_dir.return_value = False symlink_included_name.is_file.return_value = True @@ -2626,39 +2626,100 @@ def test_symlinks(self) -> None: outside_root_symlink.resolve.assert_called_once() ignored_symlink.resolve.assert_not_called() + def test_get_sources_symlink_and_force_exclude(self) -> None: + with TemporaryDirectory() as tempdir: + tmp = Path(tempdir).resolve() + actual = tmp / "actual" + actual.mkdir() + symlink = tmp / "symlink" + symlink.symlink_to(actual) + + actual_proj = actual / "project" + actual_proj.mkdir() + (actual_proj / "module.py").write_text("print('hello')", encoding="utf-8") + + symlink_proj = symlink / "project" + + with change_directory(symlink_proj): + assert_collected_sources( + src=["module.py"], + root=symlink_proj.resolve(), + expected=["module.py"], + ) + + absolute_module = symlink_proj / "module.py" + assert_collected_sources( + src=[absolute_module], + root=symlink_proj.resolve(), + expected=[absolute_module], + ) + + # a few tricky tests for force_exclude + flat_symlink = symlink_proj / "symlink_module.py" + flat_symlink.symlink_to(actual_proj / "module.py") + assert_collected_sources( + src=[flat_symlink], + root=symlink_proj.resolve(), + force_exclude=r"/symlink_module.py", + expected=[], + ) + + target = actual_proj / "target" + target.mkdir() + (target / "another.py").write_text("print('hello')", encoding="utf-8") + (symlink_proj / "nested").symlink_to(target) + + assert_collected_sources( + src=[symlink_proj / "nested" / "another.py"], + root=symlink_proj.resolve(), + force_exclude=r"nested", + expected=[], + ) + assert_collected_sources( + src=[symlink_proj / "nested" / "another.py"], + root=symlink_proj.resolve(), + force_exclude=r"target", + expected=[symlink_proj / "nested" / "another.py"], + ) + def test_get_sources_with_stdin_symlink_outside_root( self, ) -> None: path = THIS_DIR / "data" / "include_exclude_tests" stdin_filename = str(path / "b/exclude/a.py") outside_root_symlink = Path("/target_directory/a.py") + root = Path("target_dir/").resolve().absolute() with patch("pathlib.Path.resolve", return_value=outside_root_symlink): assert_collected_sources( - root=Path("target_directory/"), + root=root, src=["-"], expected=[], stdin_filename=stdin_filename, ) - @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin(self) -> None: src = ["-"] expected = ["-"] - assert_collected_sources(src, expected, include="", exclude=r"/exclude/|a\.py") + assert_collected_sources( + src, + root=THIS_DIR.resolve(), + expected=expected, + include="", + exclude=r"/exclude/|a\.py", + ) - @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin_filename(self) -> None: src = ["-"] stdin_filename = str(THIS_DIR / "data/collections.py") expected = [f"__BLACK_STDIN_FILENAME__{stdin_filename}"] assert_collected_sources( src, - expected, + root=THIS_DIR.resolve(), + expected=expected, exclude=r"/exclude/a\.py", stdin_filename=stdin_filename, ) - @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin_filename_and_exclude(self) -> None: # Exclude shouldn't exclude stdin_filename since it is mimicking the # file being passed directly. This is the same as @@ -2669,12 +2730,12 @@ def test_get_sources_with_stdin_filename_and_exclude(self) -> None: expected = [f"__BLACK_STDIN_FILENAME__{stdin_filename}"] assert_collected_sources( src, - expected, + root=THIS_DIR.resolve(), + expected=expected, exclude=r"/exclude/|a\.py", stdin_filename=stdin_filename, ) - @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin_filename_and_extend_exclude(self) -> None: # Extend exclude shouldn't exclude stdin_filename since it is mimicking the # file being passed directly. This is the same as @@ -2685,12 +2746,12 @@ def test_get_sources_with_stdin_filename_and_extend_exclude(self) -> None: expected = [f"__BLACK_STDIN_FILENAME__{stdin_filename}"] assert_collected_sources( src, - expected, + root=THIS_DIR.resolve(), + expected=expected, extend_exclude=r"/exclude/|a\.py", stdin_filename=stdin_filename, ) - @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin_filename_and_force_exclude(self) -> None: # Force exclude should exclude the file when passing it through # stdin_filename @@ -2698,27 +2759,32 @@ def test_get_sources_with_stdin_filename_and_force_exclude(self) -> None: stdin_filename = str(path / "b/exclude/a.py") assert_collected_sources( src=["-"], + root=THIS_DIR.resolve(), expected=[], force_exclude=r"/exclude/|a\.py", stdin_filename=stdin_filename, ) - @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin_filename_and_force_exclude_and_symlink( self, ) -> None: # Force exclude should exclude a symlink based on the symlink, not its target - path = THIS_DIR / "data" / "include_exclude_tests" - stdin_filename = str(path / "symlink.py") - expected = [f"__BLACK_STDIN_FILENAME__{stdin_filename}"] - target = path / "b/exclude/a.py" - with patch("pathlib.Path.resolve", return_value=target): - assert_collected_sources( - src=["-"], - expected=expected, - force_exclude=r"exclude/a\.py", - stdin_filename=stdin_filename, - ) + with TemporaryDirectory() as tempdir: + tmp = Path(tempdir).resolve() + (tmp / "exclude").mkdir() + (tmp / "exclude" / "a.py").write_text("print('hello')", encoding="utf-8") + (tmp / "symlink.py").symlink_to(tmp / "exclude" / "a.py") + + stdin_filename = str(tmp / "symlink.py") + expected = [f"__BLACK_STDIN_FILENAME__{stdin_filename}"] + with change_directory(tmp): + assert_collected_sources( + src=["-"], + root=tmp, + expected=expected, + force_exclude=r"exclude/a\.py", + stdin_filename=stdin_filename, + ) class TestDeFactoAPI: From 35e97769190d8c8fe94d9ea2d70d7d88b22a2642 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Feb 2024 06:19:25 -0800 Subject: [PATCH 52/68] Bump pre-commit/action from 3.0.0 to 3.0.1 (#4225) Bumps [pre-commit/action](https://github.com/pre-commit/action) from 3.0.0 to 3.0.1. - [Release notes](https://github.com/pre-commit/action/releases) - [Commits](https://github.com/pre-commit/action/compare/v3.0.0...v3.0.1) --- updated-dependencies: - dependency-name: pre-commit/action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8cb335bffb5..f75734400ce 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -35,7 +35,7 @@ jobs: python -m pip install tox - name: Run pre-commit hooks - uses: pre-commit/action@v3.0.0 + uses: pre-commit/action@v3.0.1 - name: Format ourselves run: | From 8af439407c051d55f3221cc93795d20bd9af49c9 Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Mon, 12 Feb 2024 08:27:50 -0600 Subject: [PATCH 53/68] fix: Don't remove comments along with parens (#4218) Signed-off-by: RedGuy12 <61329810+RedGuy12@users.noreply.github.com> --- CHANGES.md | 3 + src/black/linegen.py | 6 + tests/data/cases/comments_in_double_parens.py | 113 ++++++++++++++++++ 3 files changed, 122 insertions(+) create mode 100644 tests/data/cases/comments_in_double_parens.py diff --git a/CHANGES.md b/CHANGES.md index 4fd63ac41eb..b1a6ae3bc1c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,9 @@ +- Fixed a bug where comments where mistakenly removed along with redundant parentheses + (#4218) + ### Preview style diff --git a/src/black/linegen.py b/src/black/linegen.py index 2b7fbc492cf..cc8e41dfb20 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -1553,6 +1553,9 @@ def maybe_make_parens_invisible_in_atom( not is_type_ignore_comment_string(middle.prefix.strip()) ): first.value = "" + if first.prefix.strip(): + # Preserve comments before first paren + middle.prefix = first.prefix + middle.prefix last.value = "" maybe_make_parens_invisible_in_atom( middle, @@ -1564,6 +1567,9 @@ def maybe_make_parens_invisible_in_atom( # Strip the invisible parens from `middle` by replacing # it with the child in-between the invisible parens middle.replace(middle.children[1]) + if middle.children[-1].prefix.strip(): + # Preserve comments before last paren + last.prefix = middle.children[-1].prefix + last.prefix return False diff --git a/tests/data/cases/comments_in_double_parens.py b/tests/data/cases/comments_in_double_parens.py new file mode 100644 index 00000000000..80e7a5e5bf5 --- /dev/null +++ b/tests/data/cases/comments_in_double_parens.py @@ -0,0 +1,113 @@ +if ( + True + # sdf +): + print("hw") + +if (( + True + # sdf +)): + print("hw") + +if (( + # type: ignore + True +)): + print("hw") + +if (( + True + # type: ignore +)): + print("hw") + +if ( + # a long comment about + # the condition below + (a or b) +): + pass + +def return_true(): + return ( + ( + True # this comment gets removed accidentally + ) + ) + +def return_true(): + return (True) # this comment gets removed accidentally + + +if ( + # huh comment + (True) +): + ... + +if ( + # huh + ( + # comment + True + ) +): + ... + + +# output + +if ( + True + # sdf +): + print("hw") + +if ( + True + # sdf +): + print("hw") + +if ( + # type: ignore + True +): + print("hw") + +if ( + True + # type: ignore +): + print("hw") + +if ( + # a long comment about + # the condition below + a + or b +): + pass + + +def return_true(): + return True # this comment gets removed accidentally + + +def return_true(): + return True # this comment gets removed accidentally + + +if ( + # huh comment + True +): + ... + +if ( + # huh + # comment + True +): + ... From 6fdf8a4af28071ed1d079c01122b34c5d587207a Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 12 Feb 2024 12:11:15 -0800 Subject: [PATCH 54/68] Prepare release 24.2.0 (#4226) --- CHANGES.md | 35 +-------------------- docs/integrations/source_version_control.md | 4 +-- docs/usage_and_configuration/the_basics.md | 6 ++-- 3 files changed, 6 insertions(+), 39 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b1a6ae3bc1c..39191c1cd82 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,22 +1,14 @@ # Change Log -## Unreleased - -### Highlights - - +## 24.2.0 ### Stable style - - - Fixed a bug where comments where mistakenly removed along with redundant parentheses (#4218) ### Preview style - - - Move the `hug_parens_with_braces_and_square_brackets` feature to the unstable style due to an outstanding crash and proposed formatting tweaks (#4198) - Fixed a bug where base expressions caused inconsistent formatting of \*\* in tenary @@ -35,40 +27,15 @@ simply add an empty `[tool.black]` to the previously discovered `pyproject.toml` (#4204) -### Packaging - - - -### Parser - - - -### Performance - - - ### Output - - - Black will swallow any `SyntaxWarning`s or `DeprecationWarning`s produced by the `ast` module when performing equivalence checks (#4189) -### _Blackd_ - - - ### Integrations - - - Add a JSONSchema and provide a validate-pyproject entry-point (#4181) -### Documentation - - - ## 24.1.1 Bugfix release to fix a bug that made Black unusable on certain file systems with strict diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index 92279707d84..f24043026fa 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -8,7 +8,7 @@ Use [pre-commit](https://pre-commit.com/). Once you repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.1.1 + rev: 24.2.0 hooks: - id: black # It is recommended to specify the latest version of Python @@ -35,7 +35,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.1.1 + rev: 24.2.0 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 61c52450165..ea7a2dae5ce 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -266,8 +266,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 24.1.1 (compiled: yes) -$ black --required-version 24.1.1 -c "format = 'this'" +black, 24.2.0 (compiled: yes) +$ black --required-version 24.2.0 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -363,7 +363,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 24.1.1 +black, 24.2.0 ``` #### `--config` From d1d4fc58d3b744db35ac832cddf1d076ba5f5e7f Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 12 Feb 2024 12:46:16 -0800 Subject: [PATCH 55/68] Add new release template (#4228) --- CHANGES.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 39191c1cd82..bcf6eb44fdb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,52 @@ # Change Log +## Unreleased + +### Highlights + + + +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + + ## 24.2.0 ### Stable style From ea66d40dd7f1eaa20256e6fccaf6d7b853ccc541 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 25 Feb 2024 16:36:52 -0800 Subject: [PATCH 56/68] Update empty line documentation (#4239) Reflects status quo following #4043 Fixes #4238 --- docs/the_black_code_style/current_style.md | 39 +++++++++------------- 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index 586c79074af..68cd6175e3e 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -166,44 +166,35 @@ that in-function vertical whitespace should only be used sparingly. _Black_ will allow single empty lines inside functions, and single and double empty lines on module level left by the original editors, except when they're within parenthesized expressions. Since such expressions are always reformatted to fit minimal -space, this whitespace is lost. The other exception is that it will remove any empty -lines immediately following a statement that introduces a new indentation level. +space, this whitespace is lost. ```python # in: -def foo(): +def function( + some_argument: int, - print("All the newlines above me should be deleted!") + other_argument: int = 5, +) -> EmptyLineInParenWillBeDeleted: -if condition: - print("No newline above me!") - - print("There is a newline above me, and that's OK!") - - -class Point: - - x: int - y: int + print("One empty line above me will be kept!") +def this_is_okay_too(): + print("No empty line here") # out: -def foo(): - print("All the newlines above me should be deleted!") - - -if condition: - print("No newline above me!") +def function( + some_argument: int, + other_argument: int = 5, +) -> EmptyLineInParenWillBeDeleted: - print("There is a newline above me, and that's OK!") + print("One empty line above me will be kept!") -class Point: - x: int - y: int +def this_is_okay_too(): + print("No empty line here") ``` It will also insert proper spacing before and after function definitions. It's one line From 899002399a26348198612503ce6ca2fc298551a6 Mon Sep 17 00:00:00 2001 From: Victorien <65306057+Viicos@users.noreply.github.com> Date: Mon, 26 Feb 2024 16:49:39 +0100 Subject: [PATCH 57/68] Remove usage of `pkg_resources` in `docs/conf.py` (#4251) --- docs/conf.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 52a849d06a4..c352f98da82 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,10 +15,9 @@ import os import string +from importlib.metadata import version from pathlib import Path -from pkg_resources import get_distribution - CURRENT_DIR = Path(__file__).parent @@ -43,7 +42,7 @@ def make_pypi_svg(version: str) -> None: # Autopopulate version # The version, including alpha/beta/rc tags, but not commit hash and datestamps -release = get_distribution("black").version.split("+")[0] +release = version("black").split("+")[0] # The short X.Y version. version = release for sp in "abcfr": From 0f18001abfe50b078317395e77508b151e271624 Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Wed, 28 Feb 2024 10:27:57 -0600 Subject: [PATCH 58/68] chore: Refactor `delimiter_split()` (#4257) Signed-off-by: RedGuy12 <61329810+RedGuy12@users.noreply.github.com> --- src/black/linegen.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/black/linegen.py b/src/black/linegen.py index cc8e41dfb20..dea36d5a5d8 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -1135,6 +1135,14 @@ def _get_last_non_comment_leaf(line: Line) -> Optional[int]: return None +def _can_add_trailing_comma(leaf: Leaf, features: Collection[Feature]) -> bool: + if is_vararg(leaf, within={syms.typedargslist}): + return Feature.TRAILING_COMMA_IN_DEF in features + if is_vararg(leaf, within={syms.arglist, syms.argument}): + return Feature.TRAILING_COMMA_IN_CALL in features + return True + + def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line: if ( safe @@ -1156,10 +1164,9 @@ def delimiter_split( If the appropriate Features are given, the split will add trailing commas also in function signatures and calls that contain `*` and `**`. """ - try: - last_leaf = line.leaves[-1] - except IndexError: + if len(line.leaves) == 0: raise CannotSplit("Line empty") from None + last_leaf = line.leaves[-1] bt = line.bracket_tracker try: @@ -1167,9 +1174,11 @@ def delimiter_split( except ValueError: raise CannotSplit("No delimiters found") from None - if delimiter_priority == DOT_PRIORITY: - if bt.delimiter_count_with_priority(delimiter_priority) == 1: - raise CannotSplit("Splitting a single attribute from its owner looks wrong") + if ( + delimiter_priority == DOT_PRIORITY + and bt.delimiter_count_with_priority(delimiter_priority) == 1 + ): + raise CannotSplit("Splitting a single attribute from its owner looks wrong") current_line = Line( mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets @@ -1198,15 +1207,8 @@ def append_to_line(leaf: Leaf) -> Iterator[Line]: yield from append_to_line(comment_after) lowest_depth = min(lowest_depth, leaf.bracket_depth) - if leaf.bracket_depth == lowest_depth: - if is_vararg(leaf, within={syms.typedargslist}): - trailing_comma_safe = ( - trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features - ) - elif is_vararg(leaf, within={syms.arglist, syms.argument}): - trailing_comma_safe = ( - trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features - ) + if trailing_comma_safe and leaf.bracket_depth == lowest_depth: + trailing_comma_safe = _can_add_trailing_comma(leaf, features) if last_leaf.type == STANDALONE_COMMENT and leaf_idx == last_non_comment_leaf: current_line = _safe_add_trailing_comma( @@ -1220,6 +1222,7 @@ def append_to_line(leaf: Leaf) -> Iterator[Line]: current_line = Line( mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets ) + if current_line: current_line = _safe_add_trailing_comma( trailing_comma_safe, delimiter_priority, current_line From d0287e1f7558d97e6c0ebd6dc5bcb5b970e2bf8c Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Wed, 28 Feb 2024 15:20:46 -0600 Subject: [PATCH 59/68] Make trailing comma logic more concise (#4202) Signed-off-by: RedGuy12 <61329810+RedGuy12@users.noreply.github.com> --- src/black/linegen.py | 26 +++++++------------------- src/black/nodes.py | 28 ++++++++++++++++++++++------ 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/black/linegen.py b/src/black/linegen.py index dea36d5a5d8..4b0dc57d544 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -31,12 +31,12 @@ BRACKETS, CLOSING_BRACKETS, OPENING_BRACKETS, - RARROW, STANDALONE_COMMENT, STATEMENT, WHITESPACE, Visitor, ensure_visible, + get_annotation_type, is_arith_like, is_async_stmt_or_funcdef, is_atom_with_invisible_parens, @@ -1046,11 +1046,12 @@ def bracket_split_build_line( result.inside_brackets = True result.depth += 1 if leaves: - # Ensure a trailing comma for imports and standalone function arguments, but - # be careful not to add one after any comments or within type annotations. no_commas = ( + # Ensure a trailing comma for imports and standalone function arguments original.is_def + # Don't add one after any comments or within type annotations and opening_bracket.value == "(" + # Don't add one if there's already one there and not any( leaf.type == token.COMMA and ( @@ -1059,22 +1060,9 @@ def bracket_split_build_line( ) for leaf in leaves ) - # In particular, don't add one within a parenthesized return annotation. - # Unfortunately the indicator we're in a return annotation (RARROW) may - # be defined directly in the parent node, the parent of the parent ... - # and so on depending on how complex the return annotation is. - # This isn't perfect and there's some false negatives but they are in - # contexts were a comma is actually fine. - and not any( - node.prev_sibling.type == RARROW - for node in ( - leaves[0].parent, - getattr(leaves[0].parent, "parent", None), - ) - if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf) - ) - # Except the false negatives above for PEP 604 unions where we - # can't add the comma. + # Don't add one inside parenthesized return annotations + and get_annotation_type(leaves[0]) != "return" + # Don't add one inside PEP 604 unions and not ( leaves[0].parent and leaves[0].parent.next_sibling diff --git a/src/black/nodes.py b/src/black/nodes.py index a8869cba234..c0dca6e5783 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -3,7 +3,18 @@ """ import sys -from typing import Final, Generic, Iterator, List, Optional, Set, Tuple, TypeVar, Union +from typing import ( + Final, + Generic, + Iterator, + List, + Literal, + Optional, + Set, + Tuple, + TypeVar, + Union, +) if sys.version_info >= (3, 10): from typing import TypeGuard @@ -951,16 +962,21 @@ def is_number_token(nl: NL) -> TypeGuard[Leaf]: return nl.type == token.NUMBER -def is_part_of_annotation(leaf: Leaf) -> bool: - """Returns whether this leaf is part of type annotations.""" +def get_annotation_type(leaf: Leaf) -> Literal["return", "param", None]: + """Returns the type of annotation this leaf is part of, if any.""" ancestor = leaf.parent while ancestor is not None: if ancestor.prev_sibling and ancestor.prev_sibling.type == token.RARROW: - return True + return "return" if ancestor.parent and ancestor.parent.type == syms.tname: - return True + return "param" ancestor = ancestor.parent - return False + return None + + +def is_part_of_annotation(leaf: Leaf) -> bool: + """Returns whether this leaf is part of a type annotation.""" + return get_annotation_type(leaf) is not None def first_leaf(node: LN) -> Optional[Leaf]: From e4bfedbec2e8b10cc6b7b31442478f05db0ce06d Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Fri, 1 Mar 2024 09:02:56 -0600 Subject: [PATCH 60/68] fix: Don't move comments while splitting delimiters (#4248) Signed-off-by: RedGuy12 <61329810+RedGuy12@users.noreply.github.com> --- CHANGES.md | 2 + src/black/linegen.py | 25 ++++++++-- tests/data/cases/split_delimiter_comments.py | 51 ++++++++++++++++++++ 3 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 tests/data/cases/split_delimiter_comments.py diff --git a/CHANGES.md b/CHANGES.md index bcf6eb44fdb..e28730a3b5f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,8 @@ +- Don't move comments along with delimiters, which could cause crashes (#4248) + ### Preview style diff --git a/src/black/linegen.py b/src/black/linegen.py index 4b0dc57d544..e34ff040c73 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -12,6 +12,7 @@ from black.brackets import ( COMMA_PRIORITY, DOT_PRIORITY, + STRING_PRIORITY, get_leaves_inside_matching_brackets, max_delimiter_priority_in_atom, ) @@ -1143,6 +1144,9 @@ def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> return line +MIGRATE_COMMENT_DELIMITERS = {STRING_PRIORITY, COMMA_PRIORITY} + + @dont_increase_indentation def delimiter_split( line: Line, features: Collection[Feature], mode: Mode @@ -1187,12 +1191,22 @@ def append_to_line(leaf: Leaf) -> Iterator[Line]: ) current_line.append(leaf) + def append_comments(leaf: Leaf) -> Iterator[Line]: + for comment_after in line.comments_after(leaf): + yield from append_to_line(comment_after) + last_non_comment_leaf = _get_last_non_comment_leaf(line) for leaf_idx, leaf in enumerate(line.leaves): yield from append_to_line(leaf) - for comment_after in line.comments_after(leaf): - yield from append_to_line(comment_after) + previous_priority = leaf_idx > 0 and bt.delimiters.get( + id(line.leaves[leaf_idx - 1]) + ) + if ( + previous_priority != delimiter_priority + or delimiter_priority in MIGRATE_COMMENT_DELIMITERS + ): + yield from append_comments(leaf) lowest_depth = min(lowest_depth, leaf.bracket_depth) if trailing_comma_safe and leaf.bracket_depth == lowest_depth: @@ -1205,8 +1219,13 @@ def append_to_line(leaf: Leaf) -> Iterator[Line]: leaf_priority = bt.delimiters.get(id(leaf)) if leaf_priority == delimiter_priority: - yield current_line + if ( + leaf_idx + 1 < len(line.leaves) + and delimiter_priority not in MIGRATE_COMMENT_DELIMITERS + ): + yield from append_comments(line.leaves[leaf_idx + 1]) + yield current_line current_line = Line( mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets ) diff --git a/tests/data/cases/split_delimiter_comments.py b/tests/data/cases/split_delimiter_comments.py new file mode 100644 index 00000000000..ea29f7c034f --- /dev/null +++ b/tests/data/cases/split_delimiter_comments.py @@ -0,0 +1,51 @@ +a = ( + 1 + # type: ignore + 2 # type: ignore +) +a = ( + 1 # type: ignore + + 2 # type: ignore +) +bad_split3 = ( + "What if we have inline comments on " # First Comment + "each line of a bad split? In that " # Second Comment + "case, we should just leave it alone." # Third Comment +) +parametrize( + ( + {}, + {}, + ), + ( # foobar + {}, + {}, + ), +) + + + +# output +a = ( + 1 # type: ignore + + 2 # type: ignore +) +a = ( + 1 # type: ignore + + 2 # type: ignore +) +bad_split3 = ( + "What if we have inline comments on " # First Comment + "each line of a bad split? In that " # Second Comment + "case, we should just leave it alone." # Third Comment +) +parametrize( + ( + {}, + {}, + ), + ( # foobar + {}, + {}, + ), +) + From f03ee113c9f3dfeb477f2d4247bfb7de2e5f465c Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Sat, 2 Mar 2024 19:31:02 -0800 Subject: [PATCH 61/68] Ensure `blib2to3.pygram` is initialized before use (#4224) --- src/blib2to3/pytree.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/blib2to3/pytree.py b/src/blib2to3/pytree.py index 2a0cd6d196a..4c55d7ac77d 100644 --- a/src/blib2to3/pytree.py +++ b/src/blib2to3/pytree.py @@ -40,12 +40,15 @@ def type_repr(type_num: int) -> Union[str, int]: global _type_reprs if not _type_reprs: - from .pygram import python_symbols + from . import pygram + + if not hasattr(pygram, "python_symbols"): + pygram.initialize(cache_dir=None) # printing tokens is possible but not as useful # from .pgen2 import token // token.__dict__.items(): - for name in dir(python_symbols): - val = getattr(python_symbols, name) + for name in dir(pygram.python_symbols): + val = getattr(pygram.python_symbols, name) if type(val) == int: _type_reprs[val] = name return _type_reprs.setdefault(type_num, type_num) From 6af7d1109693c4ad3af08ecbc34649c232b47a6d Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 9 Mar 2024 17:42:29 -0800 Subject: [PATCH 62/68] Fix AST safety check false negative (#4270) Fixes #4268 Previously we would allow whitespace changes in all strings, now only in docstrings. Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- CHANGES.md | 4 ++ src/black/__init__.py | 15 ++++-- src/black/parsing.py | 42 ++++++++++++--- tests/test_black.py | 122 +++++++++++++++++++++++++++++++++++++----- 4 files changed, 156 insertions(+), 27 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e28730a3b5f..1d20a4c9210 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,10 @@ - Don't move comments along with delimiters, which could cause crashes (#4248) +- Strengthen AST safety check to catch more unsafe changes to strings. Previous versions + of Black would incorrectly format the contents of certain unusual f-strings containing + nested strings with the same quote type. Now, Black will crash on such strings until + support for the new f-string syntax is implemented. (#4270) ### Preview style diff --git a/src/black/__init__.py b/src/black/__init__.py index f82b9fec5b7..da884e6027e 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -77,8 +77,13 @@ syms, ) from black.output import color_diff, diff, dump_to_file, err, ipynb_diff, out -from black.parsing import InvalidInput # noqa F401 -from black.parsing import lib2to3_parse, parse_ast, stringify_ast +from black.parsing import ( # noqa F401 + ASTSafetyError, + InvalidInput, + lib2to3_parse, + parse_ast, + stringify_ast, +) from black.ranges import adjusted_lines, convert_unchanged_lines, parse_line_ranges from black.report import Changed, NothingChanged, Report from black.trans import iter_fexpr_spans @@ -1511,7 +1516,7 @@ def assert_equivalent(src: str, dst: str) -> None: try: src_ast = parse_ast(src) except Exception as exc: - raise AssertionError( + raise ASTSafetyError( "cannot use --safe with this file; failed to parse source file AST: " f"{exc}\n" "This could be caused by running Black with an older Python version " @@ -1522,7 +1527,7 @@ def assert_equivalent(src: str, dst: str) -> None: dst_ast = parse_ast(dst) except Exception as exc: log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst) - raise AssertionError( + raise ASTSafetyError( f"INTERNAL ERROR: Black produced invalid code: {exc}. " "Please report a bug on https://github.com/psf/black/issues. " f"This invalid output might be helpful: {log}" @@ -1532,7 +1537,7 @@ def assert_equivalent(src: str, dst: str) -> None: dst_ast_str = "\n".join(stringify_ast(dst_ast)) if src_ast_str != dst_ast_str: log = dump_to_file(diff(src_ast_str, dst_ast_str, "src", "dst")) - raise AssertionError( + raise ASTSafetyError( "INTERNAL ERROR: Black produced code that is not equivalent to the" " source. Please report a bug on " f"https://github.com/psf/black/issues. This diff might be helpful: {log}" diff --git a/src/black/parsing.py b/src/black/parsing.py index 63c5e71a0fe..aa97a8cecea 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -110,6 +110,10 @@ def lib2to3_unparse(node: Node) -> str: return code +class ASTSafetyError(Exception): + """Raised when Black's generated code is not equivalent to the old AST.""" + + def _parse_single_version( src: str, version: Tuple[int, int], *, type_comments: bool ) -> ast.AST: @@ -154,9 +158,20 @@ def _normalize(lineend: str, value: str) -> str: return normalized.strip() -def stringify_ast(node: ast.AST, depth: int = 0) -> Iterator[str]: +def stringify_ast(node: ast.AST) -> Iterator[str]: """Simple visitor generating strings to compare ASTs by content.""" + return _stringify_ast(node, []) + +def _stringify_ast_with_new_parent( + node: ast.AST, parent_stack: List[ast.AST], new_parent: ast.AST +) -> Iterator[str]: + parent_stack.append(new_parent) + yield from _stringify_ast(node, parent_stack) + parent_stack.pop() + + +def _stringify_ast(node: ast.AST, parent_stack: List[ast.AST]) -> Iterator[str]: if ( isinstance(node, ast.Constant) and isinstance(node.value, str) @@ -167,7 +182,7 @@ def stringify_ast(node: ast.AST, depth: int = 0) -> Iterator[str]: # over the kind node.kind = None - yield f"{' ' * depth}{node.__class__.__name__}(" + yield f"{' ' * len(parent_stack)}{node.__class__.__name__}(" for field in sorted(node._fields): # noqa: F402 # TypeIgnore has only one field 'lineno' which breaks this comparison @@ -179,7 +194,7 @@ def stringify_ast(node: ast.AST, depth: int = 0) -> Iterator[str]: except AttributeError: continue - yield f"{' ' * (depth + 1)}{field}=" + yield f"{' ' * (len(parent_stack) + 1)}{field}=" if isinstance(value, list): for item in value: @@ -191,13 +206,15 @@ def stringify_ast(node: ast.AST, depth: int = 0) -> Iterator[str]: and isinstance(item, ast.Tuple) ): for elt in item.elts: - yield from stringify_ast(elt, depth + 2) + yield from _stringify_ast_with_new_parent( + elt, parent_stack, node + ) elif isinstance(item, ast.AST): - yield from stringify_ast(item, depth + 2) + yield from _stringify_ast_with_new_parent(item, parent_stack, node) elif isinstance(value, ast.AST): - yield from stringify_ast(value, depth + 2) + yield from _stringify_ast_with_new_parent(value, parent_stack, node) else: normalized: object @@ -205,6 +222,12 @@ def stringify_ast(node: ast.AST, depth: int = 0) -> Iterator[str]: isinstance(node, ast.Constant) and field == "value" and isinstance(value, str) + and len(parent_stack) >= 2 + and isinstance(parent_stack[-1], ast.Expr) + and isinstance( + parent_stack[-2], + (ast.FunctionDef, ast.AsyncFunctionDef, ast.Module, ast.ClassDef), + ) ): # Constant strings may be indented across newlines, if they are # docstrings; fold spaces after newlines when comparing. Similarly, @@ -215,6 +238,9 @@ def stringify_ast(node: ast.AST, depth: int = 0) -> Iterator[str]: normalized = value.rstrip() else: normalized = value - yield f"{' ' * (depth + 2)}{normalized!r}, # {value.__class__.__name__}" + yield ( + f"{' ' * (len(parent_stack) + 1)}{normalized!r}, #" + f" {value.__class__.__name__}" + ) - yield f"{' ' * depth}) # /{node.__class__.__name__}" + yield f"{' ' * len(parent_stack)}) # /{node.__class__.__name__}" diff --git a/tests/test_black.py b/tests/test_black.py index 41f87cd16f8..96f53d5e5f3 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -46,6 +46,7 @@ from black.debug import DebugVisitor from black.mode import Mode, Preview from black.output import color_diff, diff +from black.parsing import ASTSafetyError from black.report import Report # Import other test classes @@ -1473,10 +1474,6 @@ def test_normalize_line_endings(self) -> None: ff(test_file, write_back=black.WriteBack.YES) self.assertEqual(test_file.read_bytes(), expected) - def test_assert_equivalent_different_asts(self) -> None: - with self.assertRaises(AssertionError): - black.assert_equivalent("{}", "None") - def test_root_logger_not_used_directly(self) -> None: def fail(*args: Any, **kwargs: Any) -> None: self.fail("Record created with root logger") @@ -1962,16 +1959,6 @@ def test_for_handled_unexpected_eof_error(self) -> None: exc_info.match("Cannot parse: 2:0: EOF in multi-line statement") - def test_equivalency_ast_parse_failure_includes_error(self) -> None: - with pytest.raises(AssertionError) as err: - black.assert_equivalent("a«»a = 1", "a«»a = 1") - - err.match("--safe") - # Unfortunately the SyntaxError message has changed in newer versions so we - # can't match it directly. - err.match("invalid character") - err.match(r"\(, line 1\)") - def test_line_ranges_with_code_option(self) -> None: code = textwrap.dedent("""\ if a == b: @@ -2822,6 +2809,113 @@ def test_format_file_contents(self) -> None: black.format_file_contents("x = 1\n", fast=True, mode=black.Mode()) +class TestASTSafety(BlackBaseTestCase): + def check_ast_equivalence( + self, source: str, dest: str, *, should_fail: bool = False + ) -> None: + # If we get a failure, make sure it's not because the code itself + # is invalid, since that will also cause assert_equivalent() to throw + # ASTSafetyError. + source = textwrap.dedent(source) + dest = textwrap.dedent(dest) + black.parse_ast(source) + black.parse_ast(dest) + if should_fail: + with self.assertRaises(ASTSafetyError): + black.assert_equivalent(source, dest) + else: + black.assert_equivalent(source, dest) + + def test_assert_equivalent_basic(self) -> None: + self.check_ast_equivalence("{}", "None", should_fail=True) + self.check_ast_equivalence("1+2", "1 + 2") + self.check_ast_equivalence("hi # comment", "hi") + + def test_assert_equivalent_del(self) -> None: + self.check_ast_equivalence("del (a, b)", "del a, b") + + def test_assert_equivalent_strings(self) -> None: + self.check_ast_equivalence('x = "x"', 'x = " x "', should_fail=True) + self.check_ast_equivalence( + ''' + """docstring """ + ''', + ''' + """docstring""" + ''', + ) + self.check_ast_equivalence( + ''' + """docstring """ + ''', + ''' + """ddocstring""" + ''', + should_fail=True, + ) + self.check_ast_equivalence( + ''' + class A: + """ + + docstring + + + """ + ''', + ''' + class A: + """docstring""" + ''', + ) + self.check_ast_equivalence( + """ + def f(): + " docstring " + """, + ''' + def f(): + """docstring""" + ''', + ) + self.check_ast_equivalence( + """ + async def f(): + " docstring " + """, + ''' + async def f(): + """docstring""" + ''', + ) + + def test_assert_equivalent_fstring(self) -> None: + major, minor = sys.version_info[:2] + if major < 3 or (major == 3 and minor < 12): + pytest.skip("relies on 3.12+ syntax") + # https://github.com/psf/black/issues/4268 + self.check_ast_equivalence( + """print(f"{"|".join([a,b,c])}")""", + """print(f"{" | ".join([a,b,c])}")""", + should_fail=True, + ) + self.check_ast_equivalence( + """print(f"{"|".join(['a','b','c'])}")""", + """print(f"{" | ".join(['a','b','c'])}")""", + should_fail=True, + ) + + def test_equivalency_ast_parse_failure_includes_error(self) -> None: + with pytest.raises(ASTSafetyError) as err: + black.assert_equivalent("a«»a = 1", "a«»a = 1") + + err.match("--safe") + # Unfortunately the SyntaxError message has changed in newer versions so we + # can't match it directly. + err.match("invalid character") + err.match(r"\(, line 1\)") + + try: with open(black.__file__, "r", encoding="utf-8") as _bf: black_source_lines = _bf.readlines() From e5510afc06cd238cd0cba4095283943a870a7e7b Mon Sep 17 00:00:00 2001 From: Charpy Date: Wed, 13 Mar 2024 05:14:06 +0100 Subject: [PATCH 63/68] update plugin url for Thonny (#4259) --- docs/integrations/editors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/editors.md b/docs/integrations/editors.md index 7d056160fcb..6ca1205d4c2 100644 --- a/docs/integrations/editors.md +++ b/docs/integrations/editors.md @@ -432,4 +432,4 @@ hook global WinSetOption filetype=python %{ ## Thonny -Use [Thonny-black-code-format](https://github.com/Franccisco/thonny-black-code-format). +Use [Thonny-black-formatter](https://pypi.org/project/thonny-black-formatter/). From 719e67462c80574c81a96faa144886de6da84489 Mon Sep 17 00:00:00 2001 From: Daniel Krzeminski Date: Tue, 12 Mar 2024 23:15:29 -0500 Subject: [PATCH 64/68] Fix 4227: Improve documentation for --quiet --check (#4236) Co-authored-by: Jelle Zijlstra --- CHANGES.md | 1 + docs/usage_and_configuration/the_basics.md | 3 +++ 2 files changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 1d20a4c9210..e0a034b759b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -50,6 +50,7 @@ ### Documentation +- Note what happens when `--check` is used with `--quiet` (#4236) diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index ea7a2dae5ce..7aa8a644bea 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -183,6 +183,9 @@ Don't write the files back, just return the status. _Black_ will exit with: - code 1 if some files would be reformatted; or - code 123 if there was an internal error +If used in combination with `--quiet` then only the exit code will be returned, unless +there was an internal error. + ```console $ black test.py --check All done! ✨ 🍰 ✨ From 1abcffc81816257985678f08c61584ed4287f22a Mon Sep 17 00:00:00 2001 From: Kai Sforza Date: Wed, 13 Mar 2024 00:22:10 -0400 Subject: [PATCH 65/68] Use regex where we ignore case on windows (#4252) On windows the path `FoObAR` is the same as `foobar`, so the output of `black` on a windows machine could output the path to `.gitignore` with an upper or lower-case drive letter. --- tests/test_black.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_black.py b/tests/test_black.py index 96f53d5e5f3..70884152d58 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -14,7 +14,7 @@ from contextlib import contextmanager, redirect_stderr from dataclasses import replace from io import BytesIO -from pathlib import Path +from pathlib import Path, WindowsPath from platform import system from tempfile import TemporaryDirectory from typing import ( @@ -2460,7 +2460,11 @@ def test_invalid_gitignore(self) -> None: assert result.stderr_bytes is not None gitignore = path / ".gitignore" - assert f"Could not parse {gitignore}" in result.stderr_bytes.decode() + assert re.search( + f"Could not parse {gitignore}".replace("\\", "\\\\"), + result.stderr_bytes.decode(), + re.IGNORECASE if isinstance(gitignore, WindowsPath) else 0, + ) def test_invalid_nested_gitignore(self) -> None: path = THIS_DIR / "data" / "invalid_nested_gitignore_tests" @@ -2472,7 +2476,11 @@ def test_invalid_nested_gitignore(self) -> None: assert result.stderr_bytes is not None gitignore = path / "a" / ".gitignore" - assert f"Could not parse {gitignore}" in result.stderr_bytes.decode() + assert re.search( + f"Could not parse {gitignore}".replace("\\", "\\\\"), + result.stderr_bytes.decode(), + re.IGNORECASE if isinstance(gitignore, WindowsPath) else 0, + ) def test_gitignore_that_ignores_subfolders(self) -> None: # If gitignore with */* is in root From 7b5a657285f38126bf28483478bbd9ea928077ec Mon Sep 17 00:00:00 2001 From: Samson Umezulike Date: Fri, 15 Mar 2024 19:18:47 +0100 Subject: [PATCH 66/68] Fix --line-ranges behavior when ranges are at EOF (#4273) Fixes #4264 --- CHANGES.md | 2 + src/black/__init__.py | 11 +++- src/black/ranges.py | 28 ++++++++ tests/data/cases/line_ranges_exceeding_end.py | 36 ++++++++++ .../data/cases/line_ranges_outside_source.py | 7 ++ tests/test_ranges.py | 66 ++++++++++++++++++- 6 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 tests/data/cases/line_ranges_exceeding_end.py create mode 100644 tests/data/cases/line_ranges_outside_source.py diff --git a/CHANGES.md b/CHANGES.md index e0a034b759b..c255c2a8347 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,8 @@ of Black would incorrectly format the contents of certain unusual f-strings containing nested strings with the same quote type. Now, Black will crash on such strings until support for the new f-string syntax is implemented. (#4270) +- Fixed a bug where line-ranges exceeding the last code line would not work as expected + (#4273) ### Preview style diff --git a/src/black/__init__.py b/src/black/__init__.py index da884e6027e..6f0e128f56c 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -84,7 +84,12 @@ parse_ast, stringify_ast, ) -from black.ranges import adjusted_lines, convert_unchanged_lines, parse_line_ranges +from black.ranges import ( + adjusted_lines, + convert_unchanged_lines, + parse_line_ranges, + sanitized_lines, +) from black.report import Changed, NothingChanged, Report from black.trans import iter_fexpr_spans from blib2to3.pgen2 import token @@ -1220,6 +1225,10 @@ def f( hey """ + if lines: + lines = sanitized_lines(lines, src_contents) + if not lines: + return src_contents # Nothing to format dst_contents = _format_str_once(src_contents, mode=mode, lines=lines) # Forced second pass to work around optional trailing commas (becoming # forced trailing commas on pass 2) interacting differently with optional diff --git a/src/black/ranges.py b/src/black/ranges.py index 06fa8790554..1ecaf7b0aed 100644 --- a/src/black/ranges.py +++ b/src/black/ranges.py @@ -45,6 +45,34 @@ def is_valid_line_range(lines: Tuple[int, int]) -> bool: return not lines or lines[0] <= lines[1] +def sanitized_lines( + lines: Collection[Tuple[int, int]], src_contents: str +) -> Collection[Tuple[int, int]]: + """Returns the valid line ranges for the given source. + + This removes ranges that are entirely outside the valid lines. + + Other ranges are normalized so that the start values are at least 1 and the + end values are at most the (1-based) index of the last source line. + """ + if not src_contents: + return [] + good_lines = [] + src_line_count = src_contents.count("\n") + if not src_contents.endswith("\n"): + src_line_count += 1 + for start, end in lines: + if start > src_line_count: + continue + # line-ranges are 1-based + start = max(start, 1) + if end < start: + continue + end = min(end, src_line_count) + good_lines.append((start, end)) + return good_lines + + def adjusted_lines( lines: Collection[Tuple[int, int]], original_source: str, diff --git a/tests/data/cases/line_ranges_exceeding_end.py b/tests/data/cases/line_ranges_exceeding_end.py new file mode 100644 index 00000000000..8f17491f684 --- /dev/null +++ b/tests/data/cases/line_ranges_exceeding_end.py @@ -0,0 +1,36 @@ +# flags: --line-ranges=6-1000 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +def foo1(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo2(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo3(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo4(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass + +# output +# flags: --line-ranges=6-1000 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +def foo1(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo2(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo3( + parameter_1, + parameter_2, + parameter_3, + parameter_4, + parameter_5, + parameter_6, + parameter_7, +): + pass + + +def foo4( + parameter_1, + parameter_2, + parameter_3, + parameter_4, + parameter_5, + parameter_6, + parameter_7, +): + pass diff --git a/tests/data/cases/line_ranges_outside_source.py b/tests/data/cases/line_ranges_outside_source.py new file mode 100644 index 00000000000..edec9015ff8 --- /dev/null +++ b/tests/data/cases/line_ranges_outside_source.py @@ -0,0 +1,7 @@ +# flags: --line-ranges=5000-6000 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines, in this case none. +def foo1(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo2(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo3(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo4(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass diff --git a/tests/test_ranges.py b/tests/test_ranges.py index d9fa9171a7f..a3028babf50 100644 --- a/tests/test_ranges.py +++ b/tests/test_ranges.py @@ -4,7 +4,7 @@ import pytest -from black.ranges import adjusted_lines +from black.ranges import adjusted_lines, sanitized_lines @pytest.mark.parametrize( @@ -183,3 +183,67 @@ def test_diffs(lines: List[Tuple[int, int]], adjusted: List[Tuple[int, int]]) -> 12. # last line changed """ assert adjusted == adjusted_lines(lines, original_source, modified_source) + + +@pytest.mark.parametrize( + "lines,sanitized", + [ + ( + [(1, 4)], + [(1, 4)], + ), + ( + [(2, 3)], + [(2, 3)], + ), + ( + [(2, 10)], + [(2, 4)], + ), + ( + [(0, 3)], + [(1, 3)], + ), + ( + [(0, 10)], + [(1, 4)], + ), + ( + [(-2, 3)], + [(1, 3)], + ), + ( + [(0, 0)], + [], + ), + ( + [(-2, -1)], + [], + ), + ( + [(-1, 0)], + [], + ), + ( + [(3, 1), (1, 3), (5, 6)], + [(1, 3)], + ), + ], +) +def test_sanitize( + lines: List[Tuple[int, int]], sanitized: List[Tuple[int, int]] +) -> None: + source = """\ +1. import re +2. def func(arg1, +3. arg2, arg3): +4. pass +""" + assert sanitized == sanitized_lines(lines, source) + + source_no_trailing_nl = """\ + 1. import re + 2. def func(arg1, + 3. arg2, arg3): + 4. pass""" + assert sanitized == sanitized_lines(lines, source_no_trailing_nl) From f00093672628d212b8965a8993cee8bedf5fe9b8 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 15 Mar 2024 12:06:12 -0700 Subject: [PATCH 67/68] Fix catastrophic performance in lines_with_leading_tabs_expanded() (#4278) --- CHANGES.md | 13 ++++++++++++- src/black/strings.py | 18 ++++++------------ tests/test_black.py | 12 ++++++++++++ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c255c2a8347..ffd24feb31c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,14 @@ +This release is a milestone: it fixes Black's first CVE security vulnerability. If you +run Black on untrusted input, or if you habitually put thousands of leading tab +characters in your docstrings, you are strongly encouraged to upgrade immediately to fix +[CVE-2024-21503](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-21503). + +This release also fixes a bug in Black's AST safety check that allowed Black to make +incorrect changes to certain f-strings that are valid in Python 3.12 and higher. + ### Stable style @@ -36,7 +44,10 @@ ### Performance - +- Fix catastrophic performance on docstrings that contain large numbers of leading tab + characters. This fixes + [CVE-2024-21503](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-21503). + (#4278) ### Output diff --git a/src/black/strings.py b/src/black/strings.py index 0e0f968824b..baa88162844 100644 --- a/src/black/strings.py +++ b/src/black/strings.py @@ -14,7 +14,6 @@ STRING_PREFIX_RE: Final = re.compile( r"^([" + STRING_PREFIX_CHARS + r"]*)(.*)$", re.DOTALL ) -FIRST_NON_WHITESPACE_RE: Final = re.compile(r"\s*\t+\s*(\S)") UNICODE_ESCAPE_RE: Final = re.compile( r"(?P\\+)(?P" r"(u(?P[a-fA-F0-9]{4}))" # Character with 16-bit hex value xxxx @@ -51,18 +50,13 @@ def lines_with_leading_tabs_expanded(s: str) -> List[str]: """ lines = [] for line in s.splitlines(): - # Find the index of the first non-whitespace character after a string of - # whitespace that includes at least one tab - match = FIRST_NON_WHITESPACE_RE.match(line) - if match: - first_non_whitespace_idx = match.start(1) - - lines.append( - line[:first_non_whitespace_idx].expandtabs() - + line[first_non_whitespace_idx:] - ) - else: + stripped_line = line.lstrip() + if not stripped_line or stripped_line == line: lines.append(line) + else: + prefix_length = len(line) - len(stripped_line) + prefix = line[:prefix_length].expandtabs() + lines.append(prefix + stripped_line) if s.endswith("\n"): lines.append("") return lines diff --git a/tests/test_black.py b/tests/test_black.py index 70884152d58..ecea4a073a3 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -48,6 +48,7 @@ from black.output import color_diff, diff from black.parsing import ASTSafetyError from black.report import Report +from black.strings import lines_with_leading_tabs_expanded # Import other test classes from tests.util import ( @@ -2041,6 +2042,17 @@ def test_line_ranges_in_pyproject_toml(self) -> None: b"Cannot use line-ranges in the pyproject.toml file." in result.stderr_bytes ) + def test_lines_with_leading_tabs_expanded(self) -> None: + # See CVE-2024-21503. Mostly test that this completes in a reasonable + # time. + payload = "\t" * 10_000 + assert lines_with_leading_tabs_expanded(payload) == [payload] + + tab = " " * 8 + assert lines_with_leading_tabs_expanded("\tx") == [f"{tab}x"] + assert lines_with_leading_tabs_expanded("\t\tx") == [f"{tab}{tab}x"] + assert lines_with_leading_tabs_expanded("\tx\n y") == [f"{tab}x", " y"] + class TestCaching: def test_get_cache_dir( From 552baf822992936134cbd31a38f69c8cfe7c0f05 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 15 Mar 2024 12:17:00 -0700 Subject: [PATCH 68/68] Prepare release 24.3.0 (#4279) --- CHANGES.md | 38 ++------------------- docs/integrations/source_version_control.md | 4 +-- docs/usage_and_configuration/the_basics.md | 6 ++-- 3 files changed, 7 insertions(+), 41 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ffd24feb31c..a1d674d5b5b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,11 +1,9 @@ # Change Log -## Unreleased +## 24.3.0 ### Highlights - - This release is a milestone: it fixes Black's first CVE security vulnerability. If you run Black on untrusted input, or if you habitually put thousands of leading tab characters in your docstrings, you are strongly encouraged to upgrade immediately to fix @@ -16,32 +14,14 @@ incorrect changes to certain f-strings that are valid in Python 3.12 and higher. ### Stable style - - - Don't move comments along with delimiters, which could cause crashes (#4248) - Strengthen AST safety check to catch more unsafe changes to strings. Previous versions of Black would incorrectly format the contents of certain unusual f-strings containing nested strings with the same quote type. Now, Black will crash on such strings until support for the new f-string syntax is implemented. (#4270) -- Fixed a bug where line-ranges exceeding the last code line would not work as expected +- Fix a bug where line-ranges exceeding the last code line would not work as expected (#4273) -### Preview style - - - -### Configuration - - - -### Packaging - - - -### Parser - - - ### Performance - Fix catastrophic performance on docstrings that contain large numbers of leading tab @@ -49,23 +29,9 @@ incorrect changes to certain f-strings that are valid in Python 3.12 and higher. [CVE-2024-21503](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-21503). (#4278) -### Output - - - -### _Blackd_ - - - -### Integrations - - - ### Documentation - Note what happens when `--check` is used with `--quiet` (#4236) - ## 24.2.0 diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index f24043026fa..541debdc556 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -8,7 +8,7 @@ Use [pre-commit](https://pre-commit.com/). Once you repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.2.0 + rev: 24.3.0 hooks: - id: black # It is recommended to specify the latest version of Python @@ -35,7 +35,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.2.0 + rev: 24.3.0 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 7aa8a644bea..9ce72823850 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -269,8 +269,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 24.2.0 (compiled: yes) -$ black --required-version 24.2.0 -c "format = 'this'" +black, 24.3.0 (compiled: yes) +$ black --required-version 24.3.0 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -366,7 +366,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 24.2.0 +black, 24.3.0 ``` #### `--config`