From bea8c60e9f721a5834fe6d8b89e85b162e9296dc Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 15 Aug 2018 17:43:14 -0700 Subject: [PATCH 001/163] Switch back to develop --- Makefile | 2 +- coconut/root.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index dc1fb87a4..bd2bbaed9 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,7 @@ wipe: clean .PHONY: just-upload just-upload: python setup.py sdist bdist_wheel - pip install --upgrade twine + pip install --upgrade --ignore-installed twine twine upload dist/* .PHONY: upload diff --git a/coconut/root.py b/coconut/root.py index 83d072ee3..6a8bbe9f2 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = False +DEVELOP = True # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 096efdf2b3583a396e4c3e463324480aa2fb2b0e Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 15 Aug 2018 17:43:14 -0700 Subject: [PATCH 002/163] Switch back to develop --- Makefile | 2 +- coconut/root.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index dc1fb87a4..bd2bbaed9 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,7 @@ wipe: clean .PHONY: just-upload just-upload: python setup.py sdist bdist_wheel - pip install --upgrade twine + pip install --upgrade --ignore-installed twine twine upload dist/* .PHONY: upload diff --git a/coconut/root.py b/coconut/root.py index 83d072ee3..6a8bbe9f2 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = False +DEVELOP = True # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From a64abfe036936714f4146c318bf99dba5d54ac99 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 16 Aug 2018 17:36:28 -0700 Subject: [PATCH 003/163] Upgrade sphinx --- coconut/constants.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/coconut/constants.py b/coconut/constants.py index a7e1bd13c..a6d31f817 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -175,13 +175,14 @@ def checksum(data): # we can't upgrade this; it breaks on Python 2 "ipython": (5, 4), # don't upgrade these; they break on master - "sphinx": (1, 5, 1), + "sphinx": (1, 7, 4), "sphinx_bootstrap_theme": (0, 4), } version_strictly = ( "pyparsing", "ipython", + "prompt_toolkit", "sphinx", "sphinx_bootstrap_theme", ) From 19cf0bcdef5709da6e7797c9b782d55e4fe299d7 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 16 Aug 2018 17:44:48 -0700 Subject: [PATCH 004/163] Turn off develop --- coconut/root.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/root.py b/coconut/root.py index 6a8bbe9f2..83d072ee3 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = True +DEVELOP = False # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 41537c6563b326d02a9ff90b42a56bdcecb3883f Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 15 Aug 2018 17:43:14 -0700 Subject: [PATCH 005/163] Switch back to develop --- Makefile | 2 +- coconut/root.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index dc1fb87a4..bd2bbaed9 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,7 @@ wipe: clean .PHONY: just-upload just-upload: python setup.py sdist bdist_wheel - pip install --upgrade twine + pip install --upgrade --ignore-installed twine twine upload dist/* .PHONY: upload diff --git a/coconut/root.py b/coconut/root.py index 83d072ee3..6a8bbe9f2 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = False +DEVELOP = True # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 9f8a7546aad7f3854d1f76c37a7d319b31b2f953 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 16 Aug 2018 17:36:28 -0700 Subject: [PATCH 006/163] Upgrade sphinx --- coconut/constants.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/coconut/constants.py b/coconut/constants.py index a7e1bd13c..a6d31f817 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -175,13 +175,14 @@ def checksum(data): # we can't upgrade this; it breaks on Python 2 "ipython": (5, 4), # don't upgrade these; they break on master - "sphinx": (1, 5, 1), + "sphinx": (1, 7, 4), "sphinx_bootstrap_theme": (0, 4), } version_strictly = ( "pyparsing", "ipython", + "prompt_toolkit", "sphinx", "sphinx_bootstrap_theme", ) From 41770b4846264d17e2db2761d21a689c8e4932bb Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 16 Aug 2018 17:44:48 -0700 Subject: [PATCH 007/163] Turn off develop --- coconut/root.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/root.py b/coconut/root.py index 6a8bbe9f2..83d072ee3 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = True +DEVELOP = False # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 8d84228a883e9139155847c0d090b5052292f554 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 16 Aug 2018 17:47:39 -0700 Subject: [PATCH 008/163] Turn on develop --- coconut/root.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/root.py b/coconut/root.py index 83d072ee3..6a8bbe9f2 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = False +DEVELOP = True # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 69b054a789f9d497d124468bbfbfa73a7524a18d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 13 Sep 2018 23:22:06 -0700 Subject: [PATCH 009/163] Fix igetitem of filter --- coconut/compiler/templates/header.py_template | 2 +- coconut/root.py | 2 +- tests/src/cocotest/agnostic/main.coco | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 5dc383283..3f5f376b5 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -9,7 +9,7 @@ class MatchError(Exception): """Pattern-matching error. Has attributes .pattern and .value.""" __slots__ = ("pattern", "value") {def_tco}def _coconut_igetitem(iterable, index): - if isinstance(iterable, (_coconut_reversed, _coconut_map, _coconut.filter, _coconut.zip, _coconut_enumerate, _coconut_count, _coconut.abc.Sequence)): + if isinstance(iterable, (_coconut_reversed, _coconut_map, _coconut.zip, _coconut_enumerate, _coconut_count, _coconut.abc.Sequence)): return iterable[index] if not _coconut.isinstance(index, _coconut.slice): if index < 0: diff --git a/coconut/root.py b/coconut/root.py index 6a8bbe9f2..f3ae97974 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = True +DEVELOP = 2 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index a1d5c7714..52c249a61 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -487,6 +487,7 @@ def main_test(): assert True else: assert False + assert 4 == range(2,20) |> filter$(i-> i > 3) |> .$[0] return True def tco_func() = tco_func() From 5989c6ddc9272b3ebc917898117188c2e866865e Mon Sep 17 00:00:00 2001 From: eindiran Date: Thu, 11 Oct 2018 17:05:44 -0700 Subject: [PATCH 010/163] Improving docs for indexing into iterables Signed-off-by: eindiran --- DOCS.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/DOCS.md b/DOCS.md index 0337486b0..999b5b37c 100644 --- a/DOCS.md +++ b/DOCS.md @@ -342,6 +342,8 @@ a if b else c ternary left short-circuit ===================== ========================== ``` +Note that because indexing has a greater precedence than piping, expressions of the form `x |> y[0]` are equivalent to `x |> (y[0])`. + ### Lambdas Coconut provides the simple, clean `->` operator as an alternative to Python's `lambda` statements. The syntax for the `->` operator is `(parameters) -> expression` (or `parameter -> expression` for one-argument lambdas). The operator has the same precedence as the old statement, which means it will often be necessary to surround the lambda in parentheses, and is right-associative. @@ -555,6 +557,36 @@ could_be_none()?.attr[index].method() (lambda result: None if result is None else result.attr[index].method())(could_be_none()) ``` +### Indexing Sequences + +Beyond indexing standard Python sequences, Coconut supports indexing into a number of iterables, including `range` and `map`. Indexing into one of these iterables uses the same syntax as indexing into a sequence in vanilla Python. + +##### Example + +**Coconut:** +```coconut +range(0, 12, 2)[4] # 8 + +map((i->i*2), range(10))[2] # 4 +``` + +**Python:** +Can’t be done quickly without Coconut’s iterable indexing, which requires many complicated pieces. The necessary definitions in Python can be found in the Coconut header. + +##### Indexing into `filter` + +Coconut cannot index into `filter` directly, as there is no efficient way to do so. + +```coconut +range(10) |> filter$(i->i>3) |> .[0] # doesn't work +``` + +In order to make this work, you can explicitly use iterator slicing, which is less efficient in the general case: + +```coconut +range(10) |> filter$(i->i>3) |> .$[0] # works +``` + ### Unicode Alternatives Coconut supports Unicode alternatives to many different operator symbols. The Unicode alternatives are relatively straightforward, and chosen to reflect the look and/or meaning of the original symbol. From 469922ab7f47e7e99255035efed25231f01edec8 Mon Sep 17 00:00:00 2001 From: eindiran Date: Thu, 11 Oct 2018 17:55:52 -0700 Subject: [PATCH 011/163] Minor fixes to section on indexing iterables Signed-off-by: eindiran --- DOCS.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DOCS.md b/DOCS.md index 999b5b37c..83d2de342 100644 --- a/DOCS.md +++ b/DOCS.md @@ -557,9 +557,9 @@ could_be_none()?.attr[index].method() (lambda result: None if result is None else result.attr[index].method())(could_be_none()) ``` -### Indexing Sequences +### Expanded Indexing for Iterable -Beyond indexing standard Python sequences, Coconut supports indexing into a number of iterables, including `range` and `map`. Indexing into one of these iterables uses the same syntax as indexing into a sequence in vanilla Python. +Beyond indexing standard Python sequences, Coconut supports indexing into a number of iterables, including `range` and `map`, which do not support random access in Python. In Coconut, indexing into an iterable of this type uses the same syntax as indexing into a sequence in vanilla Python. ##### Example @@ -587,6 +587,8 @@ In order to make this work, you can explicitly use iterator slicing, which is le range(10) |> filter$(i->i>3) |> .$[0] # works ``` +For more information on Coconut's iterator slicing, see [here](#iterator-slicing). + ### Unicode Alternatives Coconut supports Unicode alternatives to many different operator symbols. The Unicode alternatives are relatively straightforward, and chosen to reflect the look and/or meaning of the original symbol. From c0e1810a8eca3e84331ebf5573349d1a4297d2de Mon Sep 17 00:00:00 2001 From: eindiran Date: Fri, 12 Oct 2018 10:20:53 -0700 Subject: [PATCH 012/163] Remove reference to mypy flag disabling tco Signed-off-by: eindiran --- DOCS.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/DOCS.md b/DOCS.md index 83d2de342..7ce1ff8f2 100644 --- a/DOCS.md +++ b/DOCS.md @@ -297,8 +297,6 @@ The line magic `%load_ext coconut` will load Coconut as an extension, providing Coconut has the ability to integrate with [MyPy](http://mypy-lang.org/) to provide optional static type-checking, including for all Coconut built-ins. Simply pass `--mypy` to enable MyPy integration, though be careful to pass it only as the last argument, since all arguments after `--mypy` are passed to `mypy`, not Coconut. -_Note: Since [tail call optimization](#tail-call-optimization) prevents proper type-checking, `--mypy` implicitly disables it._ - To explicitly annotate your code with types for MyPy to check, Coconut supports [Python 3 function type annotations](https://www.python.org/dev/peps/pep-0484/), [Python 3.6 variable type annotations](https://www.python.org/dev/peps/pep-0526/), and even Coconut's own [enhanced type annotation syntax](#enhanced-type-annotation). By default, all type annotations are compiled to Python-2-compatible type comments, which means it all works on any Python version. Coconut even supports `--mypy` in the interpreter, which will intelligently scan each new line of code, in the context of previous lines, for newly-introduced MyPy errors. For example: From e1e0bce964ec49be8838852ff58a7b7ce9d5fdab Mon Sep 17 00:00:00 2001 From: eindiran Date: Fri, 12 Oct 2018 22:37:24 -0700 Subject: [PATCH 013/163] Expanding docs for None-aware operators Signed-off-by: eindiran --- DOCS.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/DOCS.md b/DOCS.md index 7ce1ff8f2..3c2bc4e35 100644 --- a/DOCS.md +++ b/DOCS.md @@ -537,25 +537,67 @@ _Can't be done without a complicated iterator slicing function and inspection of ### None Coalescing -Coconut provides `??` as a `None`-coalescing operator, similar to the `??` null-coalescing operator in C#. The `None`-coalescing operator evaluates to its left operand if that operand is not `None`, otherwise its right operand. The `None`-coalescing operator is short-circuiting, such that if the left operand is not `None`, it will not evaluate the right operand. The `None`-coalescing operator has a precedence in-between infix function calls and composition pipes, and is left-associative. The in-place operator is `??=`. +Coconut provides `??` as a `None`-coalescing operator, similar to the `??` null-coalescing operator in C#. Coconut implements all of the [PEP 505](https://www.python.org/dev/peps/pep-0505/) proposal to add `None`-aware operators to Python. -Coconut also allows a single `?` before attribute access, function calling, partial application, and (iterator) indexing to short-circuit the rest of the evaluation if everything so far evaluates to `None`. Thus, `a?.b` is equivalent to `a.b if a is not None else a`. +Coconut's `None`-coalescing operator evaluates to its left operand if that operand is not `None`, otherwise its right operand. The expression `foo ?? bar` evaluates to `foo` as long as it isn't `None` and to `bar` if it is. +The `None`-coalescing operator is short-circuiting, such that if the left operand is not `None`, the right operand won't be evaluated. This allows the right operand to be a potentially expensive operation without incurring any unnecessary cost. + +The `None`-coalescing operator has a precedence in-between infix function calls and composition pipes, and is left-associative. ##### Example **Coconut:** ```coconut could_be_none() ?? calculate_default_value() -could_be_none()?.attr[index].method() ``` **Python:** ```coconut_python (lambda result: result if result is not None else calculate_default_value())(could_be_none()) +``` +#### Coalescing Assignment Operator + +The in-place assignment operator is `??=`, which allows conditionally setting a variable if it is currently `None`. + +```coconut +foo = 1 +bar = None +foo ??= 10 # foo is still 1 +bar ??= 10 # bar is now 10 +``` + +#### Other None-aware Operators + +Coconut also allows a single `?` before attribute access, function calling, partial application, and (iterator) indexing to short-circuit the rest of the evaluation if everything so far evaluates to `None`. This is also known as a "safe navigation" operator. + +When using a `None`-aware operator for member access, either for a method or an attribute, the syntax is `obj?.method()` or `obj?.attr` respectively. `obj?.attr` is equivalent to `obj.attr if obj is not None else obj`. This does not prevent an `AttributeError` if `attr` is not an attribute or method of `obj`. + +The `None`-aware indexing operator is simply `?[]`. `seq?[index]` is equivalent to `seq[index] is seq is not None else seq`. Using this operator will not prevent an `IndexError` if `index` is outside the bounds of `seq`. + +##### Example + +**Coconut:** +```coconut +could_be_none?.attr # attribute access +could_be_none?(arg) # function calling +could_be_none?.method() # method calling +could_be_none?$(arg) # partial application +could_be_none()?[0] # indexing +could_be_none()?.attr[index].method() +``` + +**Python:** +```coconut_python +import functools +(lambda result: None if result is None else result.attr)(could_be_none()) +(lambda result: None if result is None else result(arg))(could_be_none()) +(lambda result: None if result is None else result.method())(could_be_none()) +(lambda result: None if result is None else functools.partial(result, arg))(could_be_none()) +(lambda result: None if result is None else result[0])(could_be_none()) (lambda result: None if result is None else result.attr[index].method())(could_be_none()) ``` -### Expanded Indexing for Iterable +### Expanded Indexing for Iterables Beyond indexing standard Python sequences, Coconut supports indexing into a number of iterables, including `range` and `map`, which do not support random access in Python. In Coconut, indexing into an iterable of this type uses the same syntax as indexing into a sequence in vanilla Python. From e6f7e9eaca4d5a9b754b11c230370930b99e4aea Mon Sep 17 00:00:00 2001 From: eindiran Date: Fri, 12 Oct 2018 22:54:58 -0700 Subject: [PATCH 014/163] # This is a combination of 2 commits. # This is the 1st commit message: Reword intro; more on LHS on being evaled Signed-off-by: eindiran # This is the commit message #2: Fix left<->right typo; minor wording improvements Signed-off-by: eindiran --- DOCS.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/DOCS.md b/DOCS.md index 3c2bc4e35..9627d8904 100644 --- a/DOCS.md +++ b/DOCS.md @@ -537,9 +537,9 @@ _Can't be done without a complicated iterator slicing function and inspection of ### None Coalescing -Coconut provides `??` as a `None`-coalescing operator, similar to the `??` null-coalescing operator in C#. Coconut implements all of the [PEP 505](https://www.python.org/dev/peps/pep-0505/) proposal to add `None`-aware operators to Python. +Coconut provides `??` as a `None`-coalescing operator, similar to the `??` null-coalescing operator in C# and Swift. Additionally, Coconut implements all of the `None`-aware operators proposed in [PEP 505](https://www.python.org/dev/peps/pep-0505/). -Coconut's `None`-coalescing operator evaluates to its left operand if that operand is not `None`, otherwise its right operand. The expression `foo ?? bar` evaluates to `foo` as long as it isn't `None` and to `bar` if it is. +Coconut's `??` operator evaluates to its left operand if that operand is not `None`, otherwise its right operand. The expression `foo ?? bar` evaluates to `foo` as long as it isn't `None`, and to `bar` if it is. The `None`-coalescing operator is short-circuiting, such that if the left operand is not `None`, the right operand won't be evaluated. This allows the right operand to be a potentially expensive operation without incurring any unnecessary cost. The `None`-coalescing operator has a precedence in-between infix function calls and composition pipes, and is left-associative. @@ -566,13 +566,20 @@ foo ??= 10 # foo is still 1 bar ??= 10 # bar is now 10 ``` +As described with the standard `??` operator, the `None`-coalescing assignment operator will not evaluate the right hand side unless the left hand side is `None`. + +```coconut +baz = 0 +baz ??= expensive_task() # right hand side isn't evaluated +``` + #### Other None-aware Operators -Coconut also allows a single `?` before attribute access, function calling, partial application, and (iterator) indexing to short-circuit the rest of the evaluation if everything so far evaluates to `None`. This is also known as a "safe navigation" operator. +Coconut also allows a single `?` before attribute access, function calling, partial application, and (iterator) indexing to short-circuit the rest of the evaluation if everything so far evaluates to `None`. This is sometimes known as a "safe navigation" operator. When using a `None`-aware operator for member access, either for a method or an attribute, the syntax is `obj?.method()` or `obj?.attr` respectively. `obj?.attr` is equivalent to `obj.attr if obj is not None else obj`. This does not prevent an `AttributeError` if `attr` is not an attribute or method of `obj`. -The `None`-aware indexing operator is simply `?[]`. `seq?[index]` is equivalent to `seq[index] is seq is not None else seq`. Using this operator will not prevent an `IndexError` if `index` is outside the bounds of `seq`. +The `None`-aware indexing operator is used identically to normal indexing, using `?[]` instead of `[]`. `seq?[index]` is equivalent to the expression `seq[index] is seq is not None else seq`. Using this operator will not prevent an `IndexError` if `index` is outside the bounds of `seq`. ##### Example From 1b8aedad91f693474bad38e4d646e531bd320a38 Mon Sep 17 00:00:00 2001 From: eindiran Date: Tue, 16 Oct 2018 12:55:00 -0700 Subject: [PATCH 015/163] Expanding TCO/TRE docs. --- DOCS.md | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/DOCS.md b/DOCS.md index 9627d8904..2a064a460 100644 --- a/DOCS.md +++ b/DOCS.md @@ -1256,8 +1256,6 @@ _Note: Tail call optimization (though not tail recursion elimination) will work If you are encountering a `RuntimeError` due to maximum recursion depth, it is highly recommended that you rewrite your function to meet either the criteria above for tail call optimization, or the corresponding criteria for [`recursive_iterator`](#recursive-iterator), either of which should prevent such errors. -_Note: Tail call optimization (though not tail recursion elimination) will be turned off if you pass the `--no-tco` command-line option, which is useful if you are having trouble reading your tracebacks and/or need maximum performance._ - ##### Example **Coconut:** @@ -1286,6 +1284,28 @@ _Showcases tail call optimization._ **Python:** _Can't be done without rewriting the function(s)._ +#### --no-tco flag +_Note: Tail call optimization will be turned off if you pass the `--no-tco` command-line option, which is useful if you are having trouble reading your tracebacks and/or need maximum performance. `--no-tco` does not disable tail recursion elimination. +This is because tail recursion elimination is usually faster than doing nothing, while other types of tail call optimization are usually slower than doing nothing. +Tail recursion elimination results in a big performance win because Python has a fairly large function call overhead. By unwinding a recursive function, far fewer function calls need to be made. +When the `--no-tco` flag is disabled, Coconut will attempt to do all types of tail call optimizations, handling non-recursive tail calls, split pattern-matching functions, mutual recursion, and tail recursion. When the `--no-tco` flag is enabled, Coconut will no longer perform any tail call optimizations other than tail recursion elimination. + +#### Tail Recursion Elimination and Python lambdas + +Coconut does not perform tail recursion elimination in functions that utilize lambdas in their tail call. This is because of the way that Python handles lambdas. +Each lambda stores a pointer to the namespace enclosing it, rather than a copy of the namespace. Thus, if the Coconut compiler tries to recycle anything in the namespace that produced the lambda, which needs to be done for TRE, the lambda can be changed retroactively. +A simple example demonstrating this behavior in Python3 is below: + +```python +x = 1 +foo = lambda: x +print(foo()) # 1 +x = 2 # Directly alter the values in the namespace enclosing foo +print(foo()) # 2 (!) +``` + +Because this could have unintended and potentially damaging consequences, Coconut opts to not perform TRE on any function with a lambda in its tail call. + ### Assignment Functions Coconut allows for assignment function definition that automatically returns the last line of the function body. An assignment function is constructed by substituting `=` for `:` after the function definition line. Thus, the syntax for assignment function definition is either From 3d74d37251b3b24ef7f231e97a292d8fb2bbb91a Mon Sep 17 00:00:00 2001 From: eindiran Date: Tue, 16 Oct 2018 12:59:06 -0700 Subject: [PATCH 016/163] Minor cleanup of TCO section --- DOCS.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DOCS.md b/DOCS.md index 2a064a460..c562fad6c 100644 --- a/DOCS.md +++ b/DOCS.md @@ -1285,7 +1285,9 @@ _Showcases tail call optimization._ _Can't be done without rewriting the function(s)._ #### --no-tco flag -_Note: Tail call optimization will be turned off if you pass the `--no-tco` command-line option, which is useful if you are having trouble reading your tracebacks and/or need maximum performance. `--no-tco` does not disable tail recursion elimination. +_Note: Tail call optimization will be turned off if you pass the `--no-tco` command-line option, which is useful if you are having trouble reading your tracebacks and/or need maximum performance._ + +`--no-tco` does not disable tail recursion elimination. This is because tail recursion elimination is usually faster than doing nothing, while other types of tail call optimization are usually slower than doing nothing. Tail recursion elimination results in a big performance win because Python has a fairly large function call overhead. By unwinding a recursive function, far fewer function calls need to be made. When the `--no-tco` flag is disabled, Coconut will attempt to do all types of tail call optimizations, handling non-recursive tail calls, split pattern-matching functions, mutual recursion, and tail recursion. When the `--no-tco` flag is enabled, Coconut will no longer perform any tail call optimizations other than tail recursion elimination. @@ -1294,7 +1296,7 @@ When the `--no-tco` flag is disabled, Coconut will attempt to do all types of ta Coconut does not perform tail recursion elimination in functions that utilize lambdas in their tail call. This is because of the way that Python handles lambdas. Each lambda stores a pointer to the namespace enclosing it, rather than a copy of the namespace. Thus, if the Coconut compiler tries to recycle anything in the namespace that produced the lambda, which needs to be done for TRE, the lambda can be changed retroactively. -A simple example demonstrating this behavior in Python3 is below: +A simple example demonstrating this behavior in Python: ```python x = 1 From 888795278ff9209752ff16027a639930c00259e7 Mon Sep 17 00:00:00 2001 From: eindiran Date: Thu, 18 Oct 2018 12:06:24 -0700 Subject: [PATCH 017/163] Add clarification on List vs Sequence in enhanced type annotation docs --- DOCS.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/DOCS.md b/DOCS.md index c562fad6c..bb0d8e2cf 100644 --- a/DOCS.md +++ b/DOCS.md @@ -1178,6 +1178,18 @@ Additionally, Coconut adds special syntax for making type annotations easier and ``` where `typing` is the Python 3.5 built-in [`typing` module](https://docs.python.org/3/library/typing.html). +_Note: `[]` does not map onto `typing.List[]` but onto `typing.Sequence[]`._ +There are two reasons that this design choice was made. When writing in an idiomatic functional style, assignment should be rare and tuples should be common. +Using `Sequence` covers both cases, accommodating tuples and lists and preventing indexed assignment. +When an indexed assignment is attempted into a variable typed with `Sequence`, MyPy will generate an error: + +``` +foo: int[] = [0, 1, 2, 3, 4, 5] +foo[0] = 1 # MyPy error: "Unsupported target for indexed assignment" +``` + +If you want to use `List` instead (if you want to support indexed assignment), use the standard Python 3.5 variable type annotation syntax: `foo: List[]`. + ##### Example **Coconut:** From c607dadc7e3246fb80cb6582dd78b80fb84b5c35 Mon Sep 17 00:00:00 2001 From: Mikhail Burshteyn Date: Sat, 10 Nov 2018 00:27:48 +0300 Subject: [PATCH 018/163] Add __hash__ on data types The added implementation XORs the `tuple.__hash__` of the data object with the hash of the class. This should help to avoid collisions between instances of different data types with identical members. Also added a test which ensures that: * hash of a data type instance can be computed; * hashes of instances of different data types with same data differ; * hash of a data type instance differs from hash of a plain tuple. Fixes #446. --- coconut/compiler/compiler.py | 2 ++ tests/src/cocotest/agnostic/main.coco | 2 ++ 2 files changed, 4 insertions(+) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 7313349ec..21b04d71f 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -1260,6 +1260,8 @@ def data_handle(self, loc, tokens): __ne__ = _coconut.object.__ne__ def __eq__(self, other): {oind}return self.__class__ is other.__class__ and _coconut.tuple.__eq__(self, other) +{cind}def __hash__(self): + {oind}return _coconut.tuple.__hash__(self) ^ hash(self.__class__) {cind}'''.format( oind=openindent, cind=closeindent, diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index 52c249a61..8b4c3bf55 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -267,6 +267,8 @@ def main_test(): assert issubclass(abc_, object) assert isinstance(abc(10), object) assert isinstance(abc_(10), object) + assert hash(abc(10)) == hash(abc(10)) + assert hash(abc(10)) != hash(abc_(10)) != hash((10,)) class aclass assert issubclass(aclass, object) assert isinstance(aclass(), object) From e0cedaae895724dd5e238f4bd4f2d3298837fa66 Mon Sep 17 00:00:00 2001 From: Mikhail Burshteyn Date: Sat, 10 Nov 2018 02:06:11 +0300 Subject: [PATCH 019/163] Raise CoconutSyntaxError for explicit returns in assignment functions This change covers the cases like: def f(x) = return x or def f(x) = return x These previously used to raise CoconutParseErrors, and this was not obvious what should be done to fix the code. With this change the errors are raised with a meaningful error message. Fixes #354. --- coconut/compiler/grammar.py | 10 +++++++++- tests/src/extras.coco | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 7882439b1..b10d39bcd 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -487,6 +487,11 @@ def make_suite_handle(tokens): return "\n" + openindent + tokens[0] + closeindent +def invalid_return_stmt_handle(_, loc, __): + """Raise a syntax error if encountered a return statement where an implicit return is expected.""" + raise CoconutDeferredSyntaxError("Expected expression but got return statement", loc) + + def implicit_return_handle(tokens): """Add an implicit return.""" internal_assert(len(tokens) == 1, "invalid implicit return tokens", tokens) @@ -1484,7 +1489,10 @@ class Grammar(object): ) where_stmt = attach(unsafe_simple_stmt_item + keyword("where").suppress() - where_suite, where_stmt_handle) - implicit_return = attach(testlist, implicit_return_handle) + implicit_return = ( + attach(return_stmt, invalid_return_stmt_handle) + | attach(testlist, implicit_return_handle) + ) implicit_return_stmt = ( attach(implicit_return + keyword("where").suppress() - where_suite, where_stmt_handle) | condense(implicit_return + newline) diff --git a/tests/src/extras.coco b/tests/src/extras.coco index aeb5b683e..d95a46713 100644 --- a/tests/src/extras.coco +++ b/tests/src/extras.coco @@ -93,6 +93,8 @@ def main(): assert_raises(-> parse("f''"), CoconutException) assert_raises(-> parse("f$()"), CoconutSyntaxError) assert_raises(-> parse("f(*x, y)"), CoconutSyntaxError) + assert_raises(-> parse("def f(x) = return x"), CoconutSyntaxError) + assert_raises(-> parse("def f(x) =\n return x"), CoconutSyntaxError) setup(target="2.7") assert parse("from io import BytesIO", mode="debug") == "from io import BytesIO" assert_raises(-> parse("def f(*, x=None) = x"), CoconutTargetError) From 009fbbd0e16af9e6b348855a62e2d77ff407a987 Mon Sep 17 00:00:00 2001 From: Mikhail Burshteyn Date: Sat, 10 Nov 2018 00:48:39 +0300 Subject: [PATCH 020/163] Fix CoconutKernel test when running with ipykernel>=5.0.0 The test previously failed because IPyKernel.do_execute had become a coroutine, which the test did not expect. Now, if a Future is returned, it is unwrapped, otherwise (for compatibility with older versions) the return value is left as-is. Fixes #457. --- tests/src/extras.coco | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/src/extras.coco b/tests/src/extras.coco index d95a46713..afec1ccdf 100644 --- a/tests/src/extras.coco +++ b/tests/src/extras.coco @@ -32,6 +32,18 @@ def assert_raises(c, exc): else: raise AssertionError("%s failed to raise exception %s" % (c, exc)) +def unwrap_future(maybe_future): + """ + If the passed value looks like a Future, return its result, otherwise return the value unchanged. + + This is needed for the CoconutKernel test to be compatible with ipykernel version 5 and newer, + where IPyKernel.do_execute is a coroutine. + """ + + if hasattr(maybe_future, 'result') and callable(maybe_future.result): + return maybe_future.result() + return maybe_future + def main(): if IPY: import coconut.highlighter # type: ignore @@ -103,7 +115,7 @@ def main(): assert parse("def f(*, x=None) = x") if CoconutKernel is not None: k = CoconutKernel() - exec_result = k.do_execute("derp = pow$(?, 2)", False, True, {"two": "(+)(1, 1)"}, True) + exec_result = k.do_execute("derp = pow$(?, 2)", False, True, {"two": "(+)(1, 1)"}, True) |> unwrap_future assert exec_result["status"] == "ok" assert exec_result["user_expressions"]["two"]["data"]["text/plain"] == "2" assert k.do_is_complete("if abc:")["status"] == "incomplete" From 64c9edefff03da9ac6b1c679ce6bc68187de6899 Mon Sep 17 00:00:00 2001 From: Mikhail Burshteyn Date: Tue, 13 Nov 2018 01:13:11 +0300 Subject: [PATCH 021/163] Fix addpattern incorrectly catching all MatchErrors This change introduces a context stack with MatchError subclasses that pattern-matching functions must raise instead of plain MatchError. This works the following way: * Every time a function is decorated with @addpattern, a subclass of MatchError is created and stored in the local scope; * when such function is called, this subclass is put on the stack before execution of addpattern's base_func; * when a pattern-matching function is called, it checks whether the exception class at the top of the stack has been already taken; * if it wasn't, the function marks it as taken (so that no other function would take it from the stack) and will raise it instead of plain MatchError; * inside addpattern, this specific exception is caught instead of MatchError. As a result, any pattern-matching function called directly as addpattern's base_func will raise a specific subclass of MatchError but will not propagate it to any pattern-matching functions called inside. The only case when addpattern will still catch a MatchError incorrectly is when addpattern wraps a non-pattern-matching function which calls a pattern-matching function internally (because the non-pattern-matching function will not mark the exception on the context stack as taken), but this should be extremely rare because addpattern is meant only for pattern-matching functions. This is an alternative implementation of PR #435. --- coconut/compiler/compiler.py | 8 +++-- coconut/compiler/header.py | 2 +- coconut/compiler/matching.py | 6 ++++ coconut/compiler/templates/header.py_template | 30 +++++++++++++++++-- coconut/constants.py | 1 + coconut/root.py | 1 + coconut/stubs/__coconut__.pyi | 3 ++ tests/src/cocotest/agnostic/suite.coco | 9 ++++++ 8 files changed, 54 insertions(+), 6 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 21b04d71f..27657744a 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -72,6 +72,7 @@ checksum, reserved_prefix, case_check_var, + function_match_error_var, ) from coconut.exceptions import ( CoconutException, @@ -1409,14 +1410,14 @@ def dict_comp_handle(self, loc, tokens): key, val, comp = tokens return "dict(((" + key + "), (" + val + ")) " + comp + ")" - def pattern_error(self, original, loc, value_var, check_var): + def pattern_error(self, original, loc, value_var, check_var, match_error_class='_coconut_MatchError'): """Construct a pattern-matching error message.""" base_line = clean(self.reformat(getline(loc, original))) line_wrap = self.wrap_str_of(base_line) repr_wrap = self.wrap_str_of(ascii(base_line)) return ( "if not " + check_var + ":\n" + openindent - + match_err_var + ' = _coconut_MatchError("pattern-matching failed for " ' + + match_err_var + " = " + match_error_class + '("pattern-matching failed for " ' + repr_wrap + ' " in " + _coconut.repr(_coconut.repr(' + value_var + ")))\n" + match_err_var + ".pattern = " + line_wrap + "\n" + match_err_var + ".value = " + value_var @@ -1457,7 +1458,8 @@ def name_match_funcdef_handle(self, original, loc, tokens): match_check_var + " = False\n" + matcher.out() # we only include match_to_args_var here because match_to_kwargs_var is modified during matching - + self.pattern_error(original, loc, match_to_args_var, match_check_var) + closeindent + + self.pattern_error(original, loc, match_to_args_var, match_check_var, function_match_error_var) + + closeindent ) return before_docstring, after_docstring diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index b1bbf766b..8814a4de0 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -205,7 +205,7 @@ def pattern_prepender(func): ), ) - format_dict["underscore_imports"] = "_coconut, _coconut_NamedTuple, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_pipe, _coconut_star_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial".format(**format_dict) + format_dict["underscore_imports"] = "_coconut, _coconut_NamedTuple, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_pipe, _coconut_star_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error".format(**format_dict) # ._coconut_tco_func is used in main.coco, so don't remove it # here without replacing its usage there diff --git a/coconut/compiler/matching.py b/coconut/compiler/matching.py index a814dbda2..7e02e5404 100644 --- a/coconut/compiler/matching.py +++ b/coconut/compiler/matching.py @@ -33,6 +33,7 @@ closeindent, const_vars, sentinel_var, + function_match_error_var, ) from coconut.compiler.util import paren_join @@ -251,6 +252,11 @@ def match_function(self, args, kwargs, match_args=(), star_arg=None, kwd_args=() def match_in_args_kwargs(self, match_args, args, kwargs, allow_star_args=False): """Matches against args or kwargs.""" + + # before everything, pop the FunctionMatchError from context + self.add_def(function_match_error_var + " = _coconut_get_function_match_error()") + self.increment() + req_len = 0 arg_checks = {} to_match = [] # [(move_down, match, against)] diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 3f5f376b5..1f1b21265 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -399,15 +399,41 @@ def recursive_iterator(func): tee_store[key], to_return = _coconut_tee(tee_store.get(key) or func(*args, **kwargs)) return to_return return recursive_iterator_func +class _coconut_FunctionMatchErrorContext(object): + from threading import local; threadlocal_var = local(); del local + __slots__ = ('exc_class', 'taken') + def __init__(self, exc_class): + self.exc_class = exc_class + self.taken = False + def __enter__(self): + try: + self.threadlocal_var.contexts.append(self) + except AttributeError: + self.threadlocal_var.contexts = [self] + def __exit__(self, type, value, traceback): + self.threadlocal_var.contexts.pop() + @classmethod + def get(cls): + try: + ctx = cls.threadlocal_var.contexts[-1] + except (AttributeError, IndexError): + return MatchError + if not ctx.taken: + ctx.taken = True + return ctx.exc_class + return MatchError +_coconut_get_function_match_error = _coconut_FunctionMatchErrorContext.get def addpattern(base_func): """Decorator to add a new case to a pattern-matching function, where the new case is checked last.""" def pattern_adder(func): + FunctionMatchError = type(_coconut_str("MatchError"), (MatchError,), {{}}) {tco_decorator}@_coconut.functools.wraps(func) def add_pattern_func(*args, **kwargs): try: - return base_func(*args, **kwargs) - except _coconut_MatchError: + with _coconut_FunctionMatchErrorContext(FunctionMatchError): + return base_func(*args, **kwargs) + except FunctionMatchError: return {tail_call_func_args_kwargs} return add_pattern_func return pattern_adder diff --git a/coconut/constants.py b/coconut/constants.py index a6d31f817..cb05d4179 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -388,6 +388,7 @@ def checksum(data): match_temp_var = reserved_prefix + "_match_temp" match_err_var = reserved_prefix + "_match_err" case_check_var = reserved_prefix + "_case_check" +function_match_error_var = reserved_prefix + "_FunctionMatchError" wildcard = "_" # for pattern-matching diff --git a/coconut/root.py b/coconut/root.py index f3ae97974..370686161 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -41,6 +41,7 @@ PY3_HEADER = r'''from builtins import chr, filter, hex, input, int, map, object, oct, open, print, range, str, zip, filter, reversed, enumerate py_chr, py_hex, py_input, py_int, py_map, py_object, py_oct, py_open, py_print, py_range, py_str, py_zip, py_filter, py_reversed, py_enumerate = chr, hex, input, int, map, object, oct, open, print, range, str, zip, filter, reversed, enumerate +_coconut_str = str ''' PY27_HEADER = r'''from __builtin__ import chr, filter, hex, input, int, map, object, oct, open, print, range, str, zip, filter, reversed, enumerate, raw_input, xrange diff --git a/coconut/stubs/__coconut__.pyi b/coconut/stubs/__coconut__.pyi index 1cdfe34cd..4c6a33e97 100644 --- a/coconut/stubs/__coconut__.pyi +++ b/coconut/stubs/__coconut__.pyi @@ -114,6 +114,9 @@ class MatchError(Exception): ... _coconut_MatchError = MatchError +def _coconut_get_function_match_error() -> _t.Type[MatchError]: ... + + def _coconut_tco(func: _FUNC) -> _FUNC: ... def _coconut_tail_call(func, *args, **kwargs): return func(*args, **kwargs) diff --git a/tests/src/cocotest/agnostic/suite.coco b/tests/src/cocotest/agnostic/suite.coco index 6efd3487f..8ef9deca5 100644 --- a/tests/src/cocotest/agnostic/suite.coco +++ b/tests/src/cocotest/agnostic/suite.coco @@ -503,6 +503,15 @@ def suite_test(): assert join_pairs2([(1, [2]), (1, [3])]).items() |> list == [(1, [3, 2])] assert return_in_loop(10) assert methtest().meth(5) == 5 + def test_match_error_addpattern(x is int): raise MatchError() + @addpattern(test_match_error_addpattern) + def test_match_error_addpattern(x) = x + try: + test_match_error_addpattern(0) + except MatchError as err: # must not be caught inside addpattern + assert err + else: + assert False return True def tco_test(): From 5170336800ce6b8e7c4602aebcf45384ea7642fa Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 13 Nov 2018 13:49:27 -0800 Subject: [PATCH 022/163] Temporary fix for oob line num --- coconut/compiler/compiler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 27657744a..cb623f5f6 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -944,11 +944,11 @@ def ln_comment(self, ln): """Get an end line comment. CoconutInternalExceptions should always be caught and complained.""" if self.keep_lines: if not 1 <= ln <= len(self.original_lines) + 1: - raise CoconutInternalException( + complain(CoconutInternalException( "out of bounds line number", ln, "not in range [1, " + str(len(self.original_lines) + 1) + "]", - ) - elif ln == len(self.original_lines) + 1: # trim too large + )) + if ln >= len(self.original_lines) + 1: # trim too large lni = -1 else: lni = ln - 1 From af3cb0dd282cbe2da1a2f81da8288bd40d72696e Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 20 Nov 2018 14:20:23 -0800 Subject: [PATCH 023/163] Fix watcher recompilation Resolves #401. --- coconut/command/command.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index 45ea7b619..34f79b57e 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -213,15 +213,23 @@ def use_args(self, args, interact=True, original_args=None): else: dest = args.dest + source = fixpath(args.source) + if args.package or self.mypy: package = True elif args.standalone: package = False else: - package = None # auto-decide package + # auto-decide package + if os.path.isfile(source): + package = True + elif os.path.isdir(source): + package = False + else: + raise CoconutException("could not find source path", source) with self.running_jobs(exit_on_error=not args.watch): - filepaths = self.compile_path(args.source, dest, package, args.run or args.interact, args.force) + filepaths = self.compile_path(source, dest, package, args.run or args.interact, args.force) self.run_mypy(filepaths) elif ( @@ -254,7 +262,7 @@ def use_args(self, args, interact=True, original_args=None): )): self.start_prompt() if args.watch: - self.watch(args.source, dest, package, args.run, args.force) + self.watch(source, dest, package, args.run, args.force) def register_error(self, code=1, errmsg=None): """Update the exit code.""" @@ -285,19 +293,14 @@ def handling_exceptions(self): printerr(report_this_text) self.register_error(errmsg=err.__class__.__name__) - def compile_path(self, path, write=True, package=None, *args, **kwargs): + def compile_path(self, path, write=True, package=True, *args, **kwargs): """Compile a path and returns paths to compiled files.""" - path = fixpath(path) if not isinstance(write, bool): write = fixpath(write) if os.path.isfile(path): - if package is None: - package = False destpath = self.compile_file(path, write, package, *args, **kwargs) return [destpath] if destpath is not None else [] elif os.path.isdir(path): - if package is None: - package = True return self.compile_folder(path, write, package, *args, **kwargs) else: raise CoconutException("could not find source path", path) @@ -644,7 +647,7 @@ def start_jupyter(self, args): run_args = [jupyter] + args self.register_error(run_cmd(run_args, raise_errs=False), errmsg="Jupyter error") - def watch(self, source, write=True, package=None, run=False, force=False): + def watch(self, source, write=True, package=True, run=False, force=False): """Watch a source and recompiles on change.""" from coconut.command.watch import Observer, RecompilationWatcher From fc2677bc54758728a11c559181aae1785f050e50 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 3 Dec 2018 16:27:52 -0800 Subject: [PATCH 024/163] Fix auto-deciding package --- coconut/command/command.py | 4 ++-- coconut/root.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index 34f79b57e..e1e2ebb57 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -222,9 +222,9 @@ def use_args(self, args, interact=True, original_args=None): else: # auto-decide package if os.path.isfile(source): - package = True - elif os.path.isdir(source): package = False + elif os.path.isdir(source): + package = True else: raise CoconutException("could not find source path", source) diff --git a/coconut/root.py b/coconut/root.py index 370686161..a983e89c9 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 2 +DEVELOP = 3 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 722cd3754b64236f14fbbbd006480791ac62b896 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 7 Dec 2018 14:41:40 -0800 Subject: [PATCH 025/163] Fix req versioning Resolves #468. --- coconut/requirements.py | 2 +- coconut/root.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/coconut/requirements.py b/coconut/requirements.py index e83bbbdca..d7b4f9542 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -50,7 +50,7 @@ def get_reqs(which="main"): for req in all_reqs[which]: req_str = req + ">=" + ver_tuple_to_str(min_versions[req]) if req in version_strictly: - req_str += ",<" + ver_tuple_to_str(min_versions[req][:-1]) + "." + str(min_versions[req][-1] + 1) + req_str += ",<" + ver_tuple_to_str(min_versions[req][:-1] + (min_versions[req][-1] + 1,)) reqs.append(req_str) return reqs diff --git a/coconut/root.py b/coconut/root.py index a983e89c9..758536716 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 3 +DEVELOP = 4 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 53d0818bdd3f2255121574d1405d1c69a9fa6295 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 22 Dec 2018 16:55:51 -0800 Subject: [PATCH 026/163] Add addpattern def Resolves #465. --- DOCS.md | 31 ++++++-- HELP.md | 13 ++-- coconut/compiler/compiler.py | 32 +++++++-- coconut/compiler/grammar.py | 71 +++++++++++-------- coconut/compiler/header.py | 2 +- coconut/compiler/templates/header.py_template | 1 + coconut/root.py | 2 +- tests/src/cocotest/agnostic/util.coco | 38 ++++------ 8 files changed, 117 insertions(+), 73 deletions(-) diff --git a/DOCS.md b/DOCS.md index bb0d8e2cf..5bd52628c 100644 --- a/DOCS.md +++ b/DOCS.md @@ -1387,6 +1387,27 @@ range(5) |> last_two |> print **Python:** _Can't be done without a long series of checks at the top of the function. See the compiled code for the Python syntax._ +### `addpattern` Functions + +Coconut provides the `addpattern def` syntax as a shortcut for the full +```coconut +@addpattern(func) +match def func(...): + ... +``` +syntax using the [`addpattern`](#addpattern) decorator. + +##### Example + +**Coconut:** +```coconut +def factorial(0) = 1 +addpattern def factorial(n) = n * factorial(n - 1) +``` + +**Python:** +_Can't be done without a complicated decorator definition and a long series of checks for each pattern-matching. See the compiled code for the Python syntax._ + ### Infix Functions Coconut allows for infix function calling, where an expression that evaluates to a function is surrounded by backticks and then can have arguments placed in front of or behind it. Infix calling has a precedence in-between chaining and `None`-coalescing, and is left-associative. Additionally, infix notation supports a lambda as the last argument, despite lambdas having a lower precedence. Thus, ``a `func` b -> c`` is equivalent to `func(a, b -> c)`. @@ -1658,7 +1679,7 @@ def addpattern(base_func): return pattern_adder ``` -Note that the function taken by `addpattern` must be a pattern-matching function. If `addpattern` receives a non pattern-matching function, the initial function with not raise `MatchError`. Thus, if a later function was meant to be called, `addpattern` will not know that the first match failed and the correct path will never be reached. +Note that the function taken by `addpattern` must be a pattern-matching function. If `addpattern` receives a non pattern-matching function, the function with not raise `MatchError`, and `addpattern` won't be able to detect the failed match. Thus, if a later function was meant to be called, `addpattern` will not know that the first match failed and the correct path will never be reached. For example, the following code raises a `TypeError`: ```coconut @@ -1673,21 +1694,19 @@ print_type() # appears to work print_type(1) # TypeError: print_type() takes 0 positional arguments but 1 was given ``` -This can be fixed by using the `match` keyword, making `print_type()` a pattern-matching function. This will have the effect of turning any `TypeErrors` that would be raised into `MatchErrors`. They can then be handled by new `addpattern` decorated functions as needed. - +This can be fixed by using either the `match` or `addpattern` keyword. For example: ```coconut match def print_type(): print("Received no arguments.") -@addpattern(print_type) -def print_type(x is int): +addpattern def print_type(x is int): print("Received an int.") print_type(1) # Works as expected print_type("This is a string.") # Raises MatchError ``` -`addpattern` can be used without the `match` keyword if the function it receives is an [assignment function](#assignment-functions) as seen in the example below: +The last case in an `addpattern` function, however, doesn't have to be a pattern-matching function if it is intended to catch all remaining cases. ##### Example diff --git a/HELP.md b/HELP.md index 766f2b92f..6297f414e 100644 --- a/HELP.md +++ b/HELP.md @@ -352,12 +352,11 @@ def factorial(n): raise TypeError("the argument to factorial must be an integer >= 0") ``` -By making use of the [Coconut built-in `addpattern`](DOCS.html#addpattern), we can take that from three indentation levels down to one. Take a look: +By making use of the [Coconut `addpattern` syntax](DOCS.html#addpattern), we can take that from three indentation levels down to one. Take a look: ``` def factorial(0) = 1 -@addpattern(factorial) -def factorial(n is int if n > 0) = +addpattern def factorial(n is int if n > 0) = """Compute n! where n is an integer >= 0.""" range(1, n+1) |> reduce$(*) @@ -373,14 +372,13 @@ First, assignment function notation. This one's pretty straightforward. If a fun Second, pattern-matching function definition. Pattern-matching function definition does exactly that—pattern-matches against all the arguments that are passed to the function. Unlike normal function definition, however, if the pattern doesn't match (if for example the wrong number of arguments are passed), your function will raise a `MatchError`. Finally, like destructuring assignment, if you want to be more explicit about using pattern-matching function definition, you can add a `match` before the `def`. -Third, `addpattern`. `addpattern` takes one argument, a previously-defined pattern-matching function, and returns a decorator that decorates a new pattern-matching function by adding the new pattern as an additional case to the old patterns. Thus, `addpattern` can be thought of as doing exactly what it says—it adds a new pattern to an existing pattern-matching function. +Third, `addpattern`. `addpattern` creates a new pattern-matching function by adding the new pattern as an additional case to the old pattern-matching function it is replacing. Thus, `addpattern` can be thought of as doing exactly what it says—it adds a new pattern to an existing pattern-matching function. Finally, not only can we rewrite the iterative approach using `addpattern`, as we did above, we can also rewrite the recursive approach using `addpattern`, like so: ```coconut def factorial(0) = 1 -@addpattern(factorial) -def factorial(n is int if n > 0) = +addpattern def factorial(n is int if n > 0) = """Compute n! where n is an integer >= 0.""" n * factorial(n - 1) @@ -402,8 +400,7 @@ First up is `quick_sort` for lists. We're going to use a recursive `addpattern`- ```coconut def quick_sort([]) = [] -@addpattern(quick_sort) -def quick_sort([head] + tail) = +addpattern def quick_sort([head] + tail) = """Sort the input sequence using the quick sort algorithm.""" quick_sort(left) + [head] + quick_sort(right) where: left = [x for x in tail if x < head] diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index cb623f5f6..6262c5d68 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -1441,6 +1441,7 @@ def name_match_funcdef_handle(self, original, loc, tokens): func, matches, cond = tokens else: raise CoconutInternalException("invalid match function definition tokens", tokens) + matcher = Matcher(loc, match_check_var) req_args, def_args, star_arg, kwd_args, dubstar_arg = split_args_list(matches, loc) @@ -1544,9 +1545,9 @@ def stmt_lambdef_handle(self, original, loc, tokens): "def " + name + params + ":\n" + body, ) else: - params.insert(0, name) # construct match tokens + match_tokens = [name] + list(params) self.stmt_lambdas.append( - "".join(self.name_match_funcdef_handle(original, loc, params)) + "".join(self.name_match_funcdef_handle(original, loc, match_tokens)) + body, ) return name @@ -1696,18 +1697,39 @@ def decoratable_funcdef_stmt_handle(self, original, loc, tokens, is_async=False) else: raise CoconutInternalException("invalid function definition tokens", tokens) - # extract information about the function + # process tokens raw_lines = funcdef.splitlines(True) def_stmt = raw_lines.pop(0) + + # detect addpattern functions + if def_stmt.startswith("addpattern def"): + def_stmt = def_stmt[len("addpattern "):] + addpattern = True + elif def_stmt.startswith("def"): + addpattern = False + else: + raise CoconutInternalException("invalid function definition statement", def_stmt) + + # extract information about the function func_name, func_args, func_params = None, None, None with self.complain_on_err(): - func_name, func_args, func_params = parse(self.split_func_name_args_params, def_stmt) + func_name, func_args, func_params = parse(self.split_func, def_stmt) + + # handle addpattern functions + if addpattern: + if func_name is None: + raise CoconutInternalException("could not find name in addpattern function definition", def_stmt) + # binds most tightly, except for TCO + decorators += "@_coconut_addpattern(" + func_name + ")\n" + # handle dotted function definition undotted_name = None # the function __name__ if func_name is a dotted name if func_name is not None: if "." in func_name: undotted_name = func_name.rsplit(".", 1)[-1] - def_stmt = def_stmt.replace(func_name, undotted_name) + def_stmt_pre_lparen, def_stmt_post_lparen = def_stmt.split("(", 1) + def_stmt_pre_lparen = def_stmt_pre_lparen.replace(func_name, undotted_name) + def_stmt = def_stmt_pre_lparen + "(" + def_stmt_post_lparen # handle async functions if is_async: diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index b10d39bcd..6b182836a 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -632,13 +632,13 @@ def tco_return_handle(tokens): return "return _coconut_tail_call(" + tokens[0] + ", " + tokens[1][1:] # tokens[1] contains )\n -def split_func_name_args_params_handle(tokens): +def split_func_handle(tokens): """Process splitting a function into name, params, and args.""" internal_assert(len(tokens) == 2, "invalid function definition splitting tokens", tokens) - func_name = tokens[0] + func_name, func_arg_tokens = tokens func_args = [] func_params = [] - for arg in tokens[1]: + for arg in func_arg_tokens: if len(arg) > 1 and arg[0] in ("*", "**"): func_args.append(arg[1]) elif arg[0] != "*": @@ -1466,9 +1466,9 @@ class Grammar(object): + Group(match + Optional(equals.suppress() + test)) + rparen.suppress(), )) - name_match_funcdef_ref = dotted_name + lparen.suppress() + match_args_list + match_guard + rparen.suppress() - op_match_funcdef_ref = op_match_funcdef_arg + op_funcdef_name + op_match_funcdef_arg + match_guard - base_match_funcdef = trace(keyword("def").suppress() + (op_match_funcdef | name_match_funcdef)) + name_match_funcdef_ref = keyword("def").suppress() + dotted_name + lparen.suppress() + match_args_list + match_guard + rparen.suppress() + op_match_funcdef_ref = keyword("def").suppress() + op_match_funcdef_arg + op_funcdef_name + op_match_funcdef_arg + match_guard + base_match_funcdef = trace(op_match_funcdef | name_match_funcdef) def_match_funcdef = trace(attach( base_match_funcdef + colon.suppress() @@ -1481,7 +1481,12 @@ class Grammar(object): ), join_match_funcdef, )) - match_funcdef = Optional(keyword("match").suppress()) + def_match_funcdef + match_def_modifiers = trace(Optional( + # we don't suppress addpattern so its presence can be detected later + keyword("match").suppress() + Optional(keyword("addpattern")) + | keyword("addpattern") + Optional(keyword("match")).suppress(), + )) + match_funcdef = addspace(match_def_modifiers + def_match_funcdef) where_suite = colon.suppress() - Group( newline.suppress() + indent.suppress() - OneOrMore(simple_stmt) - dedent.suppress() @@ -1507,28 +1512,35 @@ class Grammar(object): condense(addspace(keyword("def") + base_funcdef) + end_func_equals) - math_funcdef_suite, math_funcdef_handle, )) - math_match_funcdef = Optional(keyword("match").suppress()) + trace(attach( - base_match_funcdef - + equals.suppress() - - Optional(docstring) - - ( - attach(implicit_return_stmt, make_suite_handle) - | newline.suppress() - indent.suppress() + math_match_funcdef = addspace( + match_def_modifiers + + trace(attach( + base_match_funcdef + + equals.suppress() - Optional(docstring) - - attach(math_funcdef_body, make_suite_handle) - - dedent.suppress() - ), - join_match_funcdef, - )) + - ( + attach(implicit_return_stmt, make_suite_handle) + | newline.suppress() - indent.suppress() + - Optional(docstring) + - attach(math_funcdef_body, make_suite_handle) + - dedent.suppress() + ), + join_match_funcdef, + )), + ) async_stmt = Forward() async_stmt_ref = addspace(keyword("async") + (with_stmt | for_stmt)) async_funcdef = keyword("async").suppress() + (funcdef | math_funcdef) - async_match_funcdef = ( - Optional(keyword("match")) + keyword("async") - | keyword("async") + Optional(keyword("match")) - ).suppress() + (def_match_funcdef | math_match_funcdef) + async_match_funcdef = addspace(trace( + # we don't suppress addpattern so its presence can be detected later + keyword("match").suppress() + keyword("addpattern") + keyword("async").suppress() + | keyword("addpattern") + keyword("match").suppress() + keyword("async").suppress() + | keyword("match").suppress() + keyword("async").suppress() + Optional(keyword("addpattern")) + | keyword("addpattern") + keyword("async").suppress() + Optional(keyword("match")).suppress() + | keyword("async").suppress() + match_def_modifiers, + ) + (def_match_funcdef | math_match_funcdef)) datadef = Forward() data_args = Group(Optional(lparen.suppress() + ZeroOrMore( @@ -1664,11 +1676,14 @@ class Grammar(object): comma + Optional(passthrough), # implicitly suppressed ))) - split_func_name_args_params = attach( - (start_marker - keyword("def")).suppress() - dotted_base_name - lparen.suppress() - - parameters_tokens - rparen.suppress(), - split_func_name_args_params_handle, - greedy=True, # this is the root in what it's used for, so might as well evaluate greedily + split_func = attach( + start_marker.suppress() + - keyword("def").suppress() + - dotted_base_name + - lparen.suppress() - parameters_tokens - rparen.suppress(), + split_func_handle, + # this is the root in what it's used for, so might as well evaluate greedily + greedy=True, ) stores_scope = ( diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 8814a4de0..176364015 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -205,7 +205,7 @@ def pattern_prepender(func): ), ) - format_dict["underscore_imports"] = "_coconut, _coconut_NamedTuple, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_pipe, _coconut_star_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error".format(**format_dict) + format_dict["underscore_imports"] = "_coconut, _coconut_NamedTuple, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_pipe, _coconut_star_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_addpattern".format(**format_dict) # ._coconut_tco_func is used in main.coco, so don't remove it # here without replacing its usage there diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 1f1b21265..b20ca1c74 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -437,6 +437,7 @@ def addpattern(base_func): return {tail_call_func_args_kwargs} return add_pattern_func return pattern_adder +_coconut_addpattern = addpattern {def_prepattern}class _coconut_partial{object}: __slots__ = ("func", "_argdict", "_arglen", "_stargs", "keywords") if hasattr(_coconut.functools.partial, "__doc__"): diff --git a/coconut/root.py b/coconut/root.py index 758536716..2353f6231 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 4 +DEVELOP = 5 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/cocotest/agnostic/util.coco b/tests/src/cocotest/agnostic/util.coco index c8db5f3ef..04046dfb9 100644 --- a/tests/src/cocotest/agnostic/util.coco +++ b/tests/src/cocotest/agnostic/util.coco @@ -229,12 +229,10 @@ def is_odd(n) = is_even(n-1) def is_even_(0) = True -@addpattern(is_even_) # type: ignore -def is_even_(n) = is_odd_(n-1) +addpattern def is_even_(n) = is_odd_(n-1) # type: ignore def is_odd_(0) = False -@addpattern(is_odd_) # type: ignore -def is_odd_(n) = is_even_(n-1) +addpattern def is_odd_(n) = is_even_(n-1) # type: ignore # TCO/TRE tests: @@ -349,14 +347,13 @@ def factorial5(value): else: return None raise TypeError() + match def fact(n) = fact(n, 1) # type: ignore -@addpattern(fact) # type: ignore -match def fact(0, acc) = acc -@addpattern(fact) # type: ignore -match def fact(n, acc) = fact(n-1, acc*n) +match addpattern def fact(0, acc) = acc # type: ignore +addpattern match def fact(n, acc) = fact(n-1, acc*n) # type: ignore + def factorial(0, acc=1) = acc -@addpattern(factorial) # type: ignore -def factorial(n is int, acc=1 if n > 0) = +addpattern def factorial(n is int, acc=1 if n > 0) = # type: ignore """this is a docstring""" factorial(n-1, acc*n) @@ -629,12 +626,10 @@ except NameError: return pattern_prepender def add_int_or_str_1(x is int) = x + 1 -@addpattern(add_int_or_str_1) # type: ignore -def add_int_or_str_1(x is str) = x + "1" +addpattern def add_int_or_str_1(x is str) = x + "1" # type: ignore def coercive_add(a is int, b) = a + int(b) -@addpattern(coercive_add) # type: ignore -def coercive_add(a is str, b) = a + str(b) +addpattern def coercive_add(a is str, b) = a + str(b) # type: ignore @addpattern(ident) def still_ident(x) = @@ -647,12 +642,10 @@ def not_ident(x) = "bar" # Pattern-matching functions with guards def pattern_abs(x if x < 0) = -x -@addpattern(pattern_abs) # type: ignore -def pattern_abs(x) = x +addpattern def pattern_abs(x) = x # type: ignore def `pattern_abs_` (x) if x < 0 = -x -@addpattern(pattern_abs_) # type: ignore -def `pattern_abs_` (x) = x +addpattern def `pattern_abs_` (x) = x # type: ignore # Recursive iterator @@ -743,8 +736,7 @@ def anything_func(*args: int, **kwargs: int) -> None: pass # Enhanced Pattern-Matching def fact_(0, acc=1) = acc -@addpattern(fact_) # type: ignore -def fact_(n is int, acc=1 if n > 0) = fact_(n-1, acc*n) +addpattern def fact_(n is int, acc=1 if n > 0) = fact_(n-1, acc*n) # type: ignore def x_is_int(x is int) = x @@ -875,8 +867,7 @@ def ridiculously_recursive_(n): def fib(n if n < 2) = n @memoize() # type: ignore -@addpattern(fib) -def fib(n) = fib(n-1) + fib(n-2) +addpattern def fib(n) = fib(n-1) + fib(n-2) # type: ignore # MapReduce from collections import defaultdict @@ -887,8 +878,7 @@ join_pairs1 = reduce$((def (acc, (k, v)) -> ), ?, defaultdict(list)) def join_pairs2([]) = defaultdict(list) -@addpattern(join_pairs2) # type: ignore -def join_pairs2([(k, v)] + tail) = +addpattern def join_pairs2([(k, v)] + tail) = # type: ignore result = join_pairs2(tail) result[k] += v result From d5529dcf1a3aa730aa9717a883a01b9c3e5e48a9 Mon Sep 17 00:00:00 2001 From: Mathis Chenuet Date: Fri, 4 Jan 2019 21:13:40 +0100 Subject: [PATCH 027/163] fix mistake (swapped |> and |*>) --- HELP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HELP.md b/HELP.md index 6297f414e..a7be52f71 100644 --- a/HELP.md +++ b/HELP.md @@ -541,7 +541,7 @@ Copy, paste! The big new thing here is how to write `data` constructors. Since ` In this case, the constructor checks whether nothing but another `vector` was passed, in which case it returns that, otherwise it returns the result of passing the arguments to the underlying constructor, the form of which is `vector(*pts)`, since that is how we declared the data type. We use sequence pattern-matching to determine whether we were passed a single vector, which is just a list or tuple of patterns to match against the contents of the sequence. -The other new construct used here is the `|*>`, or star-pipe, operator, which functions exactly like the normal pipe, except that instead of calling the function with one argument, it calls it with as many arguments as there are elements in the sequence passed into it. The difference between `|*>` and `|>` is exactly analogous to the difference between `f(args)` and `f(*args)`. +The other new construct used here is the `|*>`, or star-pipe, operator, which functions exactly like the normal pipe, except that instead of calling the function with one argument, it calls it with as many arguments as there are elements in the sequence passed into it. The difference between `|>` and `|*>` is exactly analogous to the difference between `f(args)` and `f(*args)`. ### n-Vector Methods From dc626f77e4a6fcb701f02a3ee20c338c78c5f5ab Mon Sep 17 00:00:00 2001 From: studentiks Date: Mon, 7 Jan 2019 21:38:59 +0200 Subject: [PATCH 028/163] DOCS.md note on IntelliJ IDEA systax highlighting --- DOCS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/DOCS.md b/DOCS.md index 5bd52628c..783e899e1 100644 --- a/DOCS.md +++ b/DOCS.md @@ -249,6 +249,7 @@ Text editors with support for Coconut syntax highlighting are: - **Vim**: See [`coconut.vim`](https://github.com/manicmaniac/coconut.vim). - **Emacs**: See [`coconut-mode`](https://github.com/NickSeagull/coconut-mode). - **Atom**: See [`language-coconut`](https://github.com/enilsen16/language-coconut). +- **IntelliJ IDEA**: See [registering file types](https://www.jetbrains.com/help/idea/creating-and-registering-file-types.html). - Any editor that supports **Pygments** (e.g. **Spyder**): See Pygments section below. Alternatively, if none of the above work for you, you can just treat Coconut as Python. Simply set up your editor so it interprets all `.coco` files as Python and that should highlight most of your code well enough. From a3b2ef64370eabb8248956064bb15b627184f801 Mon Sep 17 00:00:00 2001 From: Gareth Booth <2gareth2@gmail.com> Date: Mon, 7 Jan 2019 13:21:47 +1000 Subject: [PATCH 029/163] Change reiterable __copy__ method to use tee instead of copy, as copy depletes the iterator --- coconut/compiler/templates/header.py_template | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index b20ca1c74..5f30e59f7 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -86,7 +86,8 @@ class reiterable{object}: def __reduce__(self): return (self.__class__, (self.iter,)) def __copy__(self): - return self.__class__(_coconut.copy.copy(self.iter)) + self.iter, new = _coconut_tee(self.iter) + return self.__class__(new) def __fmap__(self, func): return _coconut_map(func, self) class scan{object}: From 6bd7b6ddbb56451efaa80abb75b84f5d67e0049e Mon Sep 17 00:00:00 2001 From: Gareth Booth <2gareth2@gmail.com> Date: Mon, 7 Jan 2019 15:08:27 +1000 Subject: [PATCH 030/163] Added reiterable copy test --- tests/src/cocotest/agnostic/main.coco | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index 8b4c3bf55..55fbc29f2 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -433,6 +433,15 @@ def main_test(): assert a is None assert range(5) |> iter |> reiterable |> .[1] == 1 assert range(5) |> reiterable |> fmap$(-> _ + 1) |> list == [1, 2, 3, 4, 5] # type: ignore + + a: Iterable[int] = [1] :: [2] :: [3] + a = a |> reiterable + b = a |> reiterable + assert b |> list == [1, 2, 3] + assert b |> list == [1, 2, 3] + assert a |> list == [1, 2, 3] + assert a |> list == [1, 2, 3] + assert (+) ..*> (+) |> repr == " ..*> " assert scan((+), [1,2,3,4,5]) |> list == [1,3,6,10,15] assert scan((*), [1,2,3,4,5]) |> list == [1,2,6,24,120] From e72dd6683d84fc3e2ef588f3f060936b7f4d7530 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 9 Jan 2019 13:25:12 -0800 Subject: [PATCH 031/163] Clean up header --- coconut/compiler/templates/header.py_template | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 5f30e59f7..17a10eeb9 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -1,5 +1,5 @@ class _coconut{object}:{comment.everything_here_must_be_copied_to_stub_file} - import collections, copy, functools, types, itertools, operator, types, weakref + import collections, copy, functools, types, itertools, operator, types, weakref, threading {bind_lru_cache}{import_asyncio}{import_pickle} {import_OrderedDict} {import_collections_abc} @@ -86,8 +86,8 @@ class reiterable{object}: def __reduce__(self): return (self.__class__, (self.iter,)) def __copy__(self): - self.iter, new = _coconut_tee(self.iter) - return self.__class__(new) + self.iter, new_iter = _coconut_tee(self.iter) + return self.__class__(new_iter) def __fmap__(self, func): return _coconut_map(func, self) class scan{object}: @@ -401,8 +401,8 @@ def recursive_iterator(func): return to_return return recursive_iterator_func class _coconut_FunctionMatchErrorContext(object): - from threading import local; threadlocal_var = local(); del local __slots__ = ('exc_class', 'taken') + threadlocal_var = _coconut.threading.local() def __init__(self, exc_class): self.exc_class = exc_class self.taken = False From 5096708388808ff6f5b6dfe6ee54e4bc15348578 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 9 Jan 2019 13:56:06 -0800 Subject: [PATCH 032/163] Improve sentinel usage --- coconut/compiler/matching.py | 15 ++++----------- coconut/compiler/templates/header.py_template | 8 ++++---- coconut/constants.py | 3 +-- 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/coconut/compiler/matching.py b/coconut/compiler/matching.py index 7e02e5404..5ede8508c 100644 --- a/coconut/compiler/matching.py +++ b/coconut/compiler/matching.py @@ -32,7 +32,6 @@ openindent, closeindent, const_vars, - sentinel_var, function_match_error_var, ) from coconut.compiler.util import paren_join @@ -97,7 +96,6 @@ class Matcher(object): "var_index", "others", "guards", - "use_sentinel", ) def __init__(self, loc, check_var, checkdefs=None, names=None, var_index=0): @@ -116,7 +114,6 @@ def __init__(self, loc, check_var, checkdefs=None, names=None, var_index=0): self.var_index = var_index self.others = [] self.guards = [] - self.use_sentinel = False def duplicate(self): """Duplicates the matcher to others.""" @@ -361,13 +358,11 @@ def match_dict(self, tokens, item): if rest is None: self.add_check("_coconut.len(" + item + ") == " + str(len(matches))) - if matches: - self.use_sentinel = True for k, v in matches: key_var = self.get_temp_var() - self.add_def(key_var + " = " + item + ".get(" + k + ", " + sentinel_var + ")") + self.add_def(key_var + " = " + item + ".get(" + k + ", _coconut_sentinel)") with self.down_a_level(): - self.add_check(key_var + " is not " + sentinel_var) + self.add_check(key_var + " is not _coconut_sentinel") self.match(v, key_var) if rest is not None and rest != wildcard: match_keys = [k for k, v in matches] @@ -529,8 +524,8 @@ def match_mstring(self, tokens, item, use_bytes=None): self.add_check(item + ".endswith(" + suffix + ")") if name != wildcard: self.add_def( - name + " = " + item + "[" + - ("" if prefix is None else "_coconut.len(" + prefix + ")") + ":" + name + " = " + item + "[" + + ("" if prefix is None else "_coconut.len(" + prefix + ")") + ":" + ("" if suffix is None else "-_coconut.len(" + suffix + ")") + "]", ) @@ -623,8 +618,6 @@ def match(self, tokens, item): def out(self): """Return pattern-matching code.""" out = "" - if self.use_sentinel: - out += sentinel_var + " = _coconut.object()\n" closes = 0 for checks, defs in self.checkdefs: if checks: diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 17a10eeb9..bd3bc3c22 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -4,6 +4,7 @@ class _coconut{object}:{comment.everything_here_must_be_copied_to_stub_file} {import_OrderedDict} {import_collections_abc} Ellipsis, Exception, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, classmethod, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, map, min, max, next, object, property, range, reversed, set, slice, str, sum, super, tuple, zip, repr{comma_bytearray} = Ellipsis, Exception, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, classmethod, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, map, min, max, next, object, property, range, reversed, set, slice, str, sum, super, tuple, zip, {static_repr}{comma_bytearray} +_coconut_sentinel = _coconut.object() {def_coconut_NamedTuple} class MatchError(Exception): """Pattern-matching error. Has attributes .pattern and .value.""" @@ -94,17 +95,16 @@ class scan{object}: """Reduce func over iterable, yielding intermediate results, optionally starting from initializer.""" __slots__ = ("func", "iter", "initializer") - empty_initializer = _coconut.object() - def __init__(self, function, iterable, initializer=empty_initializer): + def __init__(self, function, iterable, initializer=_coconut_sentinel): self.func = function self.iter = iterable self.initializer = initializer def __iter__(self): acc = self.initializer - if acc is not self.empty_initializer: + if acc is not _coconut_sentinel: yield acc for item in self.iter: - if acc is self.empty_initializer: + if acc is _coconut_sentinel: acc = item else: acc = self.func(acc, item) diff --git a/coconut/constants.py b/coconut/constants.py index cb05d4179..d1357aad2 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -377,7 +377,6 @@ def checksum(data): tre_store_var = reserved_prefix + "_recursive_func" tre_check_var = reserved_prefix + "_is_recursive" none_coalesce_var = reserved_prefix + "_none_coalesce_item" -sentinel_var = reserved_prefix + "_sentinel" func_var = reserved_prefix + "_func" # prefer Matcher.get_temp_var to proliferating more match vars here @@ -615,7 +614,7 @@ def checksum(data): "\u2192", # -> "\\*?\u21a6", # |> "\u21a4\\*?", # <| - "?", # .. + "?", # .. "\u22c5", # * "\u2191", # ** "\xf7", # / From a3021f00e18a4311cd88b4ed16ecf81695a17995 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 10 Jan 2019 15:30:24 -0800 Subject: [PATCH 033/163] Improve indentation paradigm --- coconut/compiler/compiler.py | 29 ++++++++++++++--------------- coconut/constants.py | 1 - 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 6262c5d68..44b4aab86 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -53,7 +53,6 @@ unwrapper, holds, tabideal, - tabworth, match_to_var, match_to_args_var, match_to_kwargs_var, @@ -809,20 +808,18 @@ def passthrough_proc(self, inputstring, **kwargs): return "".join(out) def leading_whitespace(self, inputstring): - """Count leading whitespace.""" - count = 0 + """Get leading whitespace.""" + leading_ws = [] for i, c in enumerate(inputstring): - if c == " ": - count += 1 - elif c == "\t": - count += tabworth - (i % tabworth) + if c in " \t": + leading_ws.append(c) else: break if self.indchar is None: self.indchar = c elif c != self.indchar: self.strict_err_or_warn("found mixing of tabs and spaces", inputstring, i) - return count + return "".join(leading_ws) def ind_proc(self, inputstring, **kwargs): """Process indentation.""" @@ -861,17 +858,19 @@ def ind_proc(self, inputstring, **kwargs): if check: raise self.make_err(CoconutSyntaxError, "illegal initial indent", line, 0, self.adjust(ln)) else: - current = 0 - elif check > current: - levels.append(current) - current = check - line = openindent + line - elif check in levels: + current = "" + elif current == check: + pass + elif check in levels: # dedent point = levels.index(check) + 1 line = closeindent * (len(levels[point:]) + 1) + line levels = levels[:point] current = levels.pop() - elif current != check: + elif check.startswith(current): # indent, since current != check + levels.append(current) + current = check + line = openindent + line + else: raise self.make_err(CoconutSyntaxError, "illegal dedent to unused indentation level", line, 0, self.adjust(ln)) new.append(line) diff --git a/coconut/constants.py b/coconut/constants.py index d1357aad2..58af16ba4 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -361,7 +361,6 @@ def checksum(data): taberrfmt = 2 # spaces to indent exceptions tabideal = 4 # spaces to indent code for displaying -tabworth = 8 # worth of \t in spaces for parsing (8 = Python standard) justify_len = 79 # ideal line length From 8b0f8316aecb1cef058db6b1bfb06525cb9b0326 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 10 Jan 2019 15:43:03 -0800 Subject: [PATCH 034/163] Improve legal indentation chars --- coconut/compiler/compiler.py | 3 ++- coconut/constants.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 44b4aab86..0a59a1e90 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -72,6 +72,7 @@ reserved_prefix, case_check_var, function_match_error_var, + legal_indent_chars, ) from coconut.exceptions import ( CoconutException, @@ -811,7 +812,7 @@ def leading_whitespace(self, inputstring): """Get leading whitespace.""" leading_ws = [] for i, c in enumerate(inputstring): - if c in " \t": + if c in legal_indent_chars: leading_ws.append(c) else: break diff --git a/coconut/constants.py b/coconut/constants.py index 58af16ba4..bcc93cf52 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -326,6 +326,8 @@ def checksum(data): if sys.getrecursionlimit() < default_recursion_limit: sys.setrecursionlimit(default_recursion_limit) +legal_indent_chars = " \t\xa0" + hash_prefix = "# __coconut_hash__ = " hash_sep = "\x00" From aac5c8a03832dd9f21b83615bec2dbd7884b591c Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 5 Feb 2019 13:52:16 -0800 Subject: [PATCH 035/163] Update to new pyparsing --- Makefile | 4 ++-- coconut/compiler/header.py | 2 +- coconut/compiler/util.py | 41 ++++++++++++++++++++++++++++---------- coconut/constants.py | 16 +++++++-------- coconut/root.py | 2 +- 5 files changed, 42 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index bd2bbaed9..8feac8946 100644 --- a/Makefile +++ b/Makefile @@ -60,8 +60,8 @@ docs: clean .PHONY: clean clean: rm -rf ./docs ./dist ./build ./tests/dest index.rst profile.json - find . -name '*.pyc' -delete - find . -name '__pycache__' -delete + -find . -name '*.pyc' -delete + -find . -name '__pycache__' -delete .PHONY: wipe wipe: clean diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 176364015..5b069b282 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -205,7 +205,7 @@ def pattern_prepender(func): ), ) - format_dict["underscore_imports"] = "_coconut, _coconut_NamedTuple, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_pipe, _coconut_star_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_addpattern".format(**format_dict) + format_dict["underscore_imports"] = "_coconut, _coconut_NamedTuple, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_pipe, _coconut_star_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_addpattern, _coconut_sentinel".format(**format_dict) # ._coconut_tco_func is used in main.coco, so don't remove it # here without replacing its usage there diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 951b30a33..e84d63df1 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -62,40 +62,59 @@ # ----------------------------------------------------------------------------------------------------------------------- +def find_new_value(value, toklist, new_toklist): + """Find the value in new_toklist that corresponds to the given value in toklist.""" + # find ParseResults by looking up their tokens + if isinstance(value, ParseResults): + if value._ParseResults__toklist == toklist: + new_value_toklist = new_toklist + else: + new_value_toklist = [] + for inner_value in value._ParseResults__toklist: + new_value_toklist.append(find_new_value(inner_value, toklist, new_toklist)) + return ParseResults(new_value_toklist) + + # find other objects by looking them up directly + try: + return new_toklist[toklist.index(value)] + except ValueError: + complain(lambda: CoconutInternalException("inefficient reevaluation of tokens: {} not in {}".format( + value, + toklist, + ))) + return evaluate_tokens(value) + + def evaluate_tokens(tokens): """Evaluate the given tokens in the computation graph.""" if isinstance(tokens, str): return tokens + elif isinstance(tokens, ParseResults): + # evaluate the list portion of the ParseResults toklist, name, asList, modal = tokens.__getnewargs__() new_toklist = [evaluate_tokens(toks) for toks in toklist] new_tokens = ParseResults(new_toklist, name, asList, modal) + # evaluate the dictionary portion of the ParseResults new_tokdict = {} for name, occurrences in tokens._ParseResults__tokdict.items(): new_occurences = [] for value, position in occurrences: - if isinstance(value, ParseResults) and value._ParseResults__toklist == toklist: - new_value = new_tokens - else: - try: - new_value = new_toklist[toklist.index(value)] - except ValueError: - complain(lambda: CoconutInternalException("inefficient reevaluation of tokens: {} not in {}".format( - value, - toklist, - ))) - new_value = evaluate_tokens(value) + new_value = find_new_value(value, toklist, new_toklist) new_occurences.append(_ParseResultsWithOffset(new_value, position)) new_tokdict[name] = occurrences new_tokens._ParseResults__accumNames.update(tokens._ParseResults__accumNames) new_tokens._ParseResults__tokdict.update(new_tokdict) return new_tokens + elif isinstance(tokens, ComputationNode): return tokens.evaluate() + elif isinstance(tokens, (list, tuple)): return [evaluate_tokens(inner_toks) for inner_toks in tokens] + else: raise CoconutInternalException("invalid computation graph tokens", tokens) diff --git a/coconut/constants.py b/coconut/constants.py index bcc93cf52..3b25cb253 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -151,23 +151,23 @@ def checksum(data): } min_versions = { - "pyparsing": (2, 2, 0), - "cPyparsing": (2, 2, 0, 1, 1), + "pyparsing": (2, 3, 1), + "cPyparsing": (2, 3, 2, 1, 0, 0), "pre-commit": (1,), - "pygments": (2, 2), - "recommonmark": (0, 4), + "pygments": (2, 3), + "recommonmark": (0, 5), "psutil": (5,), "jupyter": (1, 0), "jupyter-console": (5, 2), - "ipykernel": (4, 8), - "mypy": (0, 620), + "ipykernel": (4, 10), + "mypy": (0, 660), "prompt_toolkit": (1,), "futures": (3, 2), "backports.functools-lru-cache": (1, 5), "argparse": (1, 4), - "pytest": (3,), + "pytest": (4,), "pexpect": (4,), - "watchdog": (0, 8), + "watchdog": (0, 9), "trollius": (2, 2), "requests": (2,), # don't upgrade this; it breaks on unix diff --git a/coconut/root.py b/coconut/root.py index 2353f6231..1b9a47801 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 5 +DEVELOP = 6 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 4990b529a78f7dc7a8df02ec820809d8b46bf678 Mon Sep 17 00:00:00 2001 From: eindiran Date: Fri, 15 Feb 2019 17:18:46 -0800 Subject: [PATCH 036/163] Documentation improvements for implicit lambdas --- DOCS.md | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/DOCS.md b/DOCS.md index 783e899e1..d1dab1b8e 100644 --- a/DOCS.md +++ b/DOCS.md @@ -347,8 +347,6 @@ Note that because indexing has a greater precedence than piping, expressions of Coconut provides the simple, clean `->` operator as an alternative to Python's `lambda` statements. The syntax for the `->` operator is `(parameters) -> expression` (or `parameter -> expression` for one-argument lambdas). The operator has the same precedence as the old statement, which means it will often be necessary to surround the lambda in parentheses, and is right-associative. -Additionally, Coconut also supports an implicit usage of the `->` operator of the form `(-> expression)`, which is equivalent to `((_=None) -> expression)`, which allows an implicit lambda to be used both when no arguments are required, and when one argument (assigned to `_`) is required. - _Note: If normal lambda syntax is insufficient, Coconut also supports an extended lambda syntax in the form of [statement lambdas](#statement-lambdas)._ ##### Rationale @@ -378,6 +376,26 @@ dubsums = map(lambda x, y: 2*(x+y), range(0, 10), range(10, 20)) print(list(dubsums)) ``` +#### Implicit Lambdas + +Coconut also supports implicit lambdas, which allow a lambda to take either no arguments or a single argument. Implicit lambdas are formed with the usual Coconut lambda operator `->`, in the form `(-> expression)`. This is equivalent to `((_=None) -> expression)`. When an argument is passed to an implicit lambda, it will be assigned to `_`, replacing the default value `None`. + +Below are two examples of implicit lambdas. The first uses the implicit argument `_`, while the second does not. + +**Single Argument Example:** +```coconut +square = (-> _**2) +``` + +**No-Argument Example:** +```coconut +import random + +get_random_number = (-> random.random()) +``` + +_Note: Nesting implicit lambdas can lead to problems with the scope of the `\_` parameter to each lambda. It is recommended that nesting implicit lambdas be avoided._ + ### Partial Application Coconut uses a `$` sign right after a function's name but before the open parenthesis used to call the function to denote partial application. From 378b17ad1b68ed3f8442231e67e03dd24e3bffef Mon Sep 17 00:00:00 2001 From: Elliott Indiran Date: Fri, 15 Feb 2019 17:22:29 -0800 Subject: [PATCH 037/163] Fix incorrect escaping. --- DOCS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOCS.md b/DOCS.md index d1dab1b8e..88d0afea0 100644 --- a/DOCS.md +++ b/DOCS.md @@ -394,7 +394,7 @@ import random get_random_number = (-> random.random()) ``` -_Note: Nesting implicit lambdas can lead to problems with the scope of the `\_` parameter to each lambda. It is recommended that nesting implicit lambdas be avoided._ +_Note: Nesting implicit lambdas can lead to problems with the scope of the `_` parameter to each lambda. It is recommended that nesting implicit lambdas be avoided._ ### Partial Application From 960d796bc267220b4631beb55a109a5439fdf1b1 Mon Sep 17 00:00:00 2001 From: eindiran Date: Fri, 15 Feb 2019 23:24:36 -0800 Subject: [PATCH 038/163] Add info on type annotations & statement lambdas --- DOCS.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/DOCS.md b/DOCS.md index 783e899e1..cf1346f32 100644 --- a/DOCS.md +++ b/DOCS.md @@ -349,7 +349,7 @@ Coconut provides the simple, clean `->` operator as an alternative to Python's ` Additionally, Coconut also supports an implicit usage of the `->` operator of the form `(-> expression)`, which is equivalent to `((_=None) -> expression)`, which allows an implicit lambda to be used both when no arguments are required, and when one argument (assigned to `_`) is required. -_Note: If normal lambda syntax is insufficient, Coconut also supports an extended lambda syntax in the form of [statement lambdas](#statement-lambdas)._ +_Note: If normal lambda syntax is insufficient, Coconut also supports an extended lambda syntax in the form of [statement lambdas](#statement-lambdas). Statement lambdas support type annotations for their parameters, while standard lambdas do not._ ##### Rationale @@ -1043,6 +1043,15 @@ def _lambda(x): map(_lambda, L) ``` +#### Type annotations +Another case where statement lambdas would be used over standard lambdas is when the parameters to the lambda are typed with type annotations. Statement lambdas use the standard Python syntax for adding type annotations to their parameters: + +```coconut +f = def (c: str) -> print(c) + +g = def (a: int, b: int) -> a ** b +``` + ### Lazy Lists Coconut supports the creation of lazy lists, where the contents in the list will be treated as an iterator and not evaluated until they are needed. Lazy lists can be created in Coconut simply by simply surrounding a comma-seperated list of items with `(|` and `|)` (so-called "banana brackets") instead of `[` and `]` for a list or `(` and `)` for a tuple. From 83d70d19e2bc74de053421eb957829db873dc31a Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 16 Feb 2019 13:27:38 -0800 Subject: [PATCH 039/163] Add missing defs to stub --- coconut/stubs/__coconut__.pyi | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/coconut/stubs/__coconut__.pyi b/coconut/stubs/__coconut__.pyi index 4c6a33e97..104788dee 100644 --- a/coconut/stubs/__coconut__.pyi +++ b/coconut/stubs/__coconut__.pyi @@ -104,6 +104,9 @@ _coconut_starmap = starmap parallel_map = concurrent_map = _coconut_map = map +_coconut_sentinel = object() + + from typing import ( NamedTuple as _coconut_NamedTuple, TYPE_CHECKING, @@ -126,7 +129,7 @@ def recursive_iterator(func: _ITER_FUNC) -> _ITER_FUNC: ... def addpattern(func: _FUNC) -> _t.Callable[[_FUNC2], _t.Union[_FUNC, _FUNC2]]: ... -prepattern = addpattern +_coconut_addpattern = prepattern = addpattern @_t.overload From bf182579255d1d3a0009812cfcf0c375eb97f23c Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 16 Feb 2019 13:35:11 -0800 Subject: [PATCH 040/163] Fix TYPE_CHECKING Resolves #484. --- coconut/stubs/__coconut__.pyi | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/coconut/stubs/__coconut__.pyi b/coconut/stubs/__coconut__.pyi index 104788dee..736d2674c 100644 --- a/coconut/stubs/__coconut__.pyi +++ b/coconut/stubs/__coconut__.pyi @@ -107,10 +107,8 @@ parallel_map = concurrent_map = _coconut_map = map _coconut_sentinel = object() -from typing import ( - NamedTuple as _coconut_NamedTuple, - TYPE_CHECKING, -) +_coconut_NamedTuple = _t.NamedTuple +TYPE_CHECKING = _t.TYPE_CHECKING class MatchError(Exception): ... From b86a56ff573c21e13adec945e5982996110d2da8 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 16 Feb 2019 14:19:01 -0800 Subject: [PATCH 041/163] Fix mypy errors Resolves #463. --- coconut/compiler/compiler.py | 17 +++++++------ coconut/compiler/header.py | 16 ++++++------ coconut/compiler/templates/header.py_template | 2 +- coconut/convenience.py | 8 +++--- coconut/root.py | 2 +- coconut/stubs/__coconut__.pyi | 1 - coconut/stubs/coconut/command/command.pyi | 2 +- coconut/stubs/coconut/convenience.pyi | 25 ++++++++++++++++++- 8 files changed, 49 insertions(+), 24 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 0a59a1e90..a7170a0c2 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -1338,15 +1338,16 @@ def {arg}(self): args_tuple="(" + ", ".join(base_args) + ("," if len(base_args) == 1 else "") + ")", ) + if types: + namedtuple_call = '_coconut.typing.NamedTuple("' + name + '", [' + ", ".join( + '("' + argname + '", ' + self.wrap_typedef(types.get(i, "_coconut.typing.Any")) + ")" + for i, argname in enumerate(base_args + ([starred_arg] if starred_arg is not None else [])) + ) + "])" + else: + namedtuple_call = '_coconut.collections.namedtuple("' + name + '", "' + attr_str + '")' + out = ( - "class " + name + "(" - + ( - '_coconut.collections.namedtuple("' + name + '", "' + attr_str + '")' if not types - else '_coconut_NamedTuple("' + name + '", [' + ", ".join( - '("' + argname + '", ' + self.wrap_typedef(types.get(i, "_coconut.typing.Any")) + ")" - for i, argname in enumerate(base_args + ([starred_arg] if starred_arg is not None else [])) - ) + "])" - ) + ( + "class " + name + "(" + namedtuple_call + ( ", " + inherit if inherit is not None else ", _coconut.object" if not self.target.startswith("3") else "" diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 5b069b282..7e8604aa9 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -178,12 +178,6 @@ class you_need_to_install_trollius: pass tco_decorator="@_coconut_tco\n" + " " * 8 if not no_tco else "", tail_call_func_args_kwargs="func(*args, **kwargs)" if no_tco else "_coconut_tail_call(func, *args, **kwargs)", comma_tco=", _coconut_tail_call, _coconut_tco" if not no_tco else "", - def_coconut_NamedTuple=( - r'''def _coconut_NamedTuple(name, fields): - return _coconut.collections.namedtuple(name, [x for x, t in fields])''' - if target_info < (3, 6) - else "from typing import NamedTuple as _coconut_NamedTuple" - ), def_prepattern=( r'''def prepattern(base_func): """DEPRECATED: Use addpattern instead.""" @@ -205,7 +199,15 @@ def pattern_prepender(func): ), ) - format_dict["underscore_imports"] = "_coconut, _coconut_NamedTuple, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_pipe, _coconut_star_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_addpattern, _coconut_sentinel".format(**format_dict) + format_dict["import_typing_NamedTuple"] = _indent( + "import typing" if target_info >= (3, 6) + else '''class typing{object}: + @staticmethod + def NamedTuple(name, fields): + return _coconut.collections.namedtuple(name, [x for x, t in fields])'''.format(**format_dict), + ) + + format_dict["underscore_imports"] = "_coconut, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_pipe, _coconut_star_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_addpattern, _coconut_sentinel".format(**format_dict) # ._coconut_tco_func is used in main.coco, so don't remove it # here without replacing its usage there diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index bd3bc3c22..fcef4f520 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -3,9 +3,9 @@ class _coconut{object}:{comment.everything_here_must_be_copied_to_stub_file} {bind_lru_cache}{import_asyncio}{import_pickle} {import_OrderedDict} {import_collections_abc} +{import_typing_NamedTuple} Ellipsis, Exception, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, classmethod, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, map, min, max, next, object, property, range, reversed, set, slice, str, sum, super, tuple, zip, repr{comma_bytearray} = Ellipsis, Exception, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, classmethod, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, map, min, max, next, object, property, range, reversed, set, slice, str, sum, super, tuple, zip, {static_repr}{comma_bytearray} _coconut_sentinel = _coconut.object() -{def_coconut_NamedTuple} class MatchError(Exception): """Pattern-matching error. Has attributes .pattern and .value.""" __slots__ = ("pattern", "value") diff --git a/coconut/convenience.py b/coconut/convenience.py index fb46a5c16..86e664afa 100644 --- a/coconut/convenience.py +++ b/coconut/convenience.py @@ -117,7 +117,7 @@ def find_module(self, fullname, path=None): if fullname.startswith("."): if path is None: # we can't do a relative import if there's no package path - return None + return fullname = fullname[1:] basepaths.insert(0, path) fullpath = os.path.join(*fullname.split(".")) @@ -128,12 +128,12 @@ def find_module(self, fullname, path=None): if os.path.exists(filepath): self.run_compiler(filepath) # Coconut file was found and compiled, now let Python import it - return None + return if os.path.exists(dirpath): self.run_compiler(path) # Coconut package was found and compiled, now let Python import it - return None - return None + return + return coconut_importer = CoconutImporter() diff --git a/coconut/root.py b/coconut/root.py index 1b9a47801..5e6d44377 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 6 +DEVELOP = 7 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/coconut/stubs/__coconut__.pyi b/coconut/stubs/__coconut__.pyi index 736d2674c..851d522ad 100644 --- a/coconut/stubs/__coconut__.pyi +++ b/coconut/stubs/__coconut__.pyi @@ -107,7 +107,6 @@ parallel_map = concurrent_map = _coconut_map = map _coconut_sentinel = object() -_coconut_NamedTuple = _t.NamedTuple TYPE_CHECKING = _t.TYPE_CHECKING diff --git a/coconut/stubs/coconut/command/command.pyi b/coconut/stubs/coconut/command/command.pyi index c4c90a66a..7f47447f8 100644 --- a/coconut/stubs/coconut/command/command.pyi +++ b/coconut/stubs/coconut/command/command.pyi @@ -16,5 +16,5 @@ Description: MyPy stub file for command.py. # ----------------------------------------------------------------------------------------------------------------------- -class Command(): +class Command: ... diff --git a/coconut/stubs/coconut/convenience.pyi b/coconut/stubs/coconut/convenience.pyi index 13cbe9a20..6c6c4df82 100644 --- a/coconut/stubs/coconut/convenience.pyi +++ b/coconut/stubs/coconut/convenience.pyi @@ -24,6 +24,9 @@ from typing import ( from coconut.command.command import Command +class CoconutException(Exception): + ... + #----------------------------------------------------------------------------------------------------------------------- # COMMAND: #----------------------------------------------------------------------------------------------------------------------- @@ -35,7 +38,7 @@ CLI: Command = ... def cmd(args: Union[Text, bytes, Iterable], interact: bool) -> None: ... -VERSION: Dict[Text, Text] = ... +VERSIONS: Dict[Text, Text] = ... def version(which: Text) -> Text: ... @@ -53,3 +56,23 @@ PARSERS: Dict[Text, Callable] = ... def parse(code: Text, mode: Text) -> Text: ... + + +# ----------------------------------------------------------------------------------------------------------------------- +# IMPORTER: +# ----------------------------------------------------------------------------------------------------------------------- + + +class CoconutImporter: + ext: str + + @staticmethod + def run_compiler(path: str) -> None: ... + + def find_module(self, fullname: str, path:str=None) -> None: ... + + +coconut_importer = CoconutImporter() + + +def auto_compilation(on:bool=True) -> None: ... From 5c2758b11ca15850d46516013815c87a47d92852 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 16 Feb 2019 14:23:50 -0800 Subject: [PATCH 042/163] Backdate pytest --- coconut/constants.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/coconut/constants.py b/coconut/constants.py index 3b25cb253..cd7e6e1eb 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -165,11 +165,12 @@ def checksum(data): "futures": (3, 2), "backports.functools-lru-cache": (1, 5), "argparse": (1, 4), - "pytest": (4,), "pexpect": (4,), "watchdog": (0, 9), "trollius": (2, 2), "requests": (2,), + # don't upgrade this; it breaks on Python 2.6 + "pytest": (3,), # don't upgrade this; it breaks on unix "vprof": (0, 36), # we can't upgrade this; it breaks on Python 2 From a76782fcb9603a1f616ced36ea6316d5f4b81589 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 17 Feb 2019 15:59:21 -0800 Subject: [PATCH 043/163] Update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a26ef386f..94dc752c9 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ bin/ # Installer logs pip-log.txt pip-delete-this-directory.txt +pip-wheel-metadata/ # Unit test / coverage reports htmlcov/ From 351451a71339c9a54905f2e294d6342b93b9a516 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 18 Feb 2019 02:35:38 -0800 Subject: [PATCH 044/163] Fix importlib.reload importing --- coconut/constants.py | 2 ++ tests/src/cocotest/agnostic/main.coco | 1 + 2 files changed, 3 insertions(+) diff --git a/coconut/constants.py b/coconut/constants.py index cd7e6e1eb..a4abcc14f 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -441,6 +441,7 @@ def checksum(data): "where", ) + py3_to_py2_stdlib = { # new_name: (old_name, before_version_info) "builtins": ("__builtin__", (3,)), @@ -479,6 +480,7 @@ def checksum(data): # ./ denotes from ... import ... "io.StringIO": ("StringIO./StringIO", (2, 7)), "io.BytesIO": ("cStringIO./StringIO", (2, 7)), + "importlib.reload": ("imp./reload", (3, 4)), # third-party backports "asyncio": ("trollius", (3, 4)), } diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index 55fbc29f2..b67ecad1d 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -499,6 +499,7 @@ def main_test(): else: assert False assert 4 == range(2,20) |> filter$(i-> i > 3) |> .$[0] + from importlib import reload return True def tco_func() = tco_func() From 7873f57e4b8531845131944f65d110116898e384 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 18 Feb 2019 02:53:22 -0800 Subject: [PATCH 045/163] Bump develop version --- coconut/root.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/root.py b/coconut/root.py index 5e6d44377..a6f4a809c 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 7 +DEVELOP = 8 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From b959c9e9df6810a855eb211b5214a8c239fc9404 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 23 Feb 2019 22:06:42 -0800 Subject: [PATCH 046/163] Fix py version testing --- tests/main_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/main_test.py b/tests/main_test.py index d7f5fcebf..3223ff330 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -136,7 +136,7 @@ def call_coconut(args, **kwargs): args = ["--jobs", "sys"] + args if "--mypy" in args and "check_mypy" not in kwargs: kwargs["check_mypy"] = True - call(["coconut"] + args, **kwargs) + call([sys.executable, "-m", "coconut"] + args, **kwargs) def comp(path=None, folder=None, file=None, args=[], **kwargs): @@ -164,7 +164,7 @@ def remove_when_done(path): shutil.rmtree(path) elif os.path.isfile(path): os.remove(path) - except OSError as err: + except OSError: logger.display_exc() From 29e2734d7f6ee2472a464356b82916f359b2f6f2 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 23 Feb 2019 22:34:23 -0800 Subject: [PATCH 047/163] Improve header --- coconut/compiler/header.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 7e8604aa9..ac7f2cec1 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -288,7 +288,7 @@ def getheader(which, target="", use_hash=None, no_tco=False, strict=False): _coconut_sys.path.insert(0, _coconut_file_path) from __coconut__ import {underscore_imports} from __coconut__ import * -_coconut_sys.path.remove(_coconut_file_path) +_coconut_sys.path.pop(0) '''.format(**format_dict) + section("Compiled Coconut") From 530ff08537dcdf3fb8887022bf7216bb7e6c55ca Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 24 Feb 2019 00:34:43 -0800 Subject: [PATCH 048/163] Improve package compilation Resolves #488. --- .pre-commit-config.yaml | 17 +++-- coconut/command/command.py | 71 +++++++++++++------ coconut/command/util.py | 44 ++++++------ coconut/compiler/compiler.py | 13 ++-- coconut/compiler/header.py | 21 ++++-- coconut/compiler/templates/header.py_template | 2 +- coconut/root.py | 2 +- 7 files changed, 104 insertions(+), 66 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7f6ff561a..f0d16ab85 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: git://github.com/pre-commit/pre-commit-hooks.git - rev: v1.3.0 + rev: v2.1.0 hooks: - id: check-byte-order-marker - id: check-merge-conflict @@ -15,17 +15,20 @@ repos: - id: pretty-format-json args: - --autofix - - id: autopep8-wrapper + - id: flake8 + args: + - --ignore=W503,E501,E265,E402,F405,E305 +- repo: https://github.com/pre-commit/mirrors-autopep8 + rev: v1.4.3 + hooks: + - id: autopep8 args: - --in-place - --aggressive - --aggressive - --experimental - - --ignore=W503,E501,E722 - - id: flake8 - args: - - --ignore=W503,E501,E265,E402,F405,E305 + - --ignore=W503,E501,E722,E402 - repo: https://github.com/asottile/add-trailing-comma - rev: v0.6.4 + rev: v0.7.2 hooks: - id: add-trailing-comma diff --git a/coconut/command/command.py b/coconut/command/command.py index e1e2ebb57..338d7f993 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -33,7 +33,10 @@ CoconutException, CoconutInternalException, ) -from coconut.terminal import logger, printerr +from coconut.terminal import ( + logger, + printerr, +) from coconut.constants import ( fixpath, code_exts, @@ -359,13 +362,18 @@ def compile(self, codepath, destpath=None, package=False, run=False, force=False code = readfile(opened) if destpath is not None: + destpath = fixpath(destpath) destdir = os.path.dirname(destpath) if not os.path.exists(destdir): os.makedirs(destdir) if package is True: - self.create_package(destdir) + package_level = self.get_package_level(codepath) + if package_level == 0: + self.create_package(destdir) + else: + package_level = -1 - foundhash = None if force else self.has_hash_of(destpath, code, package) + foundhash = None if force else self.has_hash_of(destpath, code, package_level) if foundhash: if show_unchanged: logger.show_tabulated("Left unchanged", showpath(destpath), "(pass --force to override).") @@ -377,13 +385,6 @@ def compile(self, codepath, destpath=None, package=False, run=False, force=False else: logger.show_tabulated("Compiling", showpath(codepath), "...") - if package is True: - compile_method = "parse_package" - elif package is False: - compile_method = "parse_file" - else: - raise CoconutInternalException("invalid value for package", package) - def callback(compiled): if destpath is None: logger.show_tabulated("Compiled", showpath(codepath), "without writing to file.") @@ -399,7 +400,40 @@ def callback(compiled): else: self.execute_file(destpath) - self.submit_comp_job(codepath, callback, compile_method, code) + if package is True: + self.submit_comp_job(codepath, callback, "parse_package", code, package_level=package_level) + elif package is False: + self.submit_comp_job(codepath, callback, "parse_file", code) + else: + raise CoconutInternalException("invalid value for package", package) + + def get_package_level(self, codepath): + """Get the relative level to the base directory of the package.""" + package_level = -1 + check_dir = os.path.dirname(os.path.abspath(codepath)) + while check_dir: + has_init = False + for ext in code_exts: + init_file = os.path.join(check_dir, "__init__" + ext) + if os.path.exists(init_file): + has_init = True + break + if has_init: + package_level += 1 + check_dir = os.path.dirname(check_dir) + else: + break + if package_level < 0: + logger.warn("missing __init__" + code_exts[0] + " in package", check_dir) + package_level = 0 + return package_level + return 0 + + def create_package(self, dirpath): + """Set up a package directory.""" + filepath = os.path.join(dirpath, "__coconut__.py") + with openfile(filepath, "w") as opened: + writefile(opened, self.comp.getheader("__coconut__")) def submit_comp_job(self, path, callback, method, *args, **kwargs): """Submits a job on self.comp to be run in parallel.""" @@ -453,22 +487,15 @@ def running_jobs(self, exit_on_error=True): if exit_on_error: self.exit_on_error() - def create_package(self, dirpath): - """Set up a package directory.""" - dirpath = fixpath(dirpath) - filepath = os.path.join(dirpath, "__coconut__.py") - with openfile(filepath, "w") as opened: - writefile(opened, self.comp.getheader("__coconut__")) - - def has_hash_of(self, destpath, code, package): + def has_hash_of(self, destpath, code, package_level): """Determine if a file has the hash of the code.""" if destpath is not None and os.path.isfile(destpath): with openfile(destpath, "r") as opened: compiled = readfile(opened) hashash = gethash(compiled) - if hashash is not None and hashash == self.comp.genhash(package, code): - return compiled - return None + if hashash is not None and hashash == self.comp.genhash(code, package_level): + return True + return False def get_input(self, more=False): """Prompt for code input.""" diff --git a/coconut/command/util.py b/coconut/command/util.py index 33ffbdec7..dfc0b222a 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -23,20 +23,20 @@ import os import traceback import subprocess -if PY26: - import imp -from functools import partial -from copy import copy -from contextlib import contextmanager from select import select -if not PY26: - import runpy -try: - # just importing readline improves built-in input() - import readline # NOQA -except ImportError: - pass +from contextlib import contextmanager +from copy import copy +from functools import partial +from coconut.terminal import ( + logger, + complain, +) +from coconut.exceptions import ( + CoconutException, + get_encoding, + internal_assert, +) from coconut.constants import ( fixpath, default_encoding, @@ -60,16 +60,16 @@ WINDOWS, PY34, ) -from coconut.exceptions import ( - CoconutException, - get_encoding, - internal_assert, -) -from coconut.terminal import ( - logger, - complain, -) +if PY26: + import imp +if not PY26: + import runpy +try: + # just importing readline improves built-in input() + import readline # NOQA +except ImportError: + pass if PY34: from importlib import reload else: @@ -438,7 +438,7 @@ def __init__(self, comp=None, exit=sys.exit, store=False, path=None): self.vars = self.build_vars(path) self.stored = [] if store else None if comp is not None: - self.store(comp.getheader("package")) + self.store(comp.getheader("package:0")) self.run(comp.getheader("code"), store=False) self.fix_pickle() diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index a7170a0c2..4b2a64605 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -365,13 +365,13 @@ def __reduce__(self): """Return pickling information.""" return (Compiler, (self.target, self.strict, self.minify, self.line_numbers, self.keep_lines, self.no_tco)) - def genhash(self, package, code): + def genhash(self, code, package_level=-1): """Generate a hash from code.""" return hex(checksum( hash_sep.join( str(item) for item in (VERSION_STR,) + self.__reduce__()[1] - + (package, code) + + (package_level, code) ).encode(default_encoding), )) @@ -1988,7 +1988,7 @@ def parse_single(self, inputstring): def parse_file(self, inputstring, addhash=True): """Parse file code.""" if addhash: - use_hash = self.genhash(False, inputstring) + use_hash = self.genhash(inputstring) else: use_hash = None return self.parse(inputstring, self.file_parser, {"nl_at_eof_check": True}, {"header": "file", "use_hash": use_hash}) @@ -1997,13 +1997,14 @@ def parse_exec(self, inputstring): """Parse exec code.""" return self.parse(inputstring, self.file_parser, {}, {"header": "file", "initial": "none"}) - def parse_package(self, inputstring, addhash=True): + def parse_package(self, inputstring, package_level=0, addhash=True): """Parse package code.""" + internal_assert(package_level >= 0, "invalid package level", package_level) if addhash: - use_hash = self.genhash(True, inputstring) + use_hash = self.genhash(inputstring, package_level) else: use_hash = None - return self.parse(inputstring, self.file_parser, {"nl_at_eof_check": True}, {"header": "package", "use_hash": use_hash}) + return self.parse(inputstring, self.file_parser, {"nl_at_eof_check": True}, {"header": "package:" + str(package_level), "use_hash": use_hash}) def parse_block(self, inputstring): """Parse block code.""" diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index ac7f2cec1..6dc70958a 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -241,12 +241,15 @@ def tail_call_optimized_func(*args, **kwargs): # ----------------------------------------------------------------------------------------------------------------------- -allowed_headers = ("none", "initial", "__coconut__", "package", "sys", "code", "file") - - def getheader(which, target="", use_hash=None, no_tco=False, strict=False): """Generate the specified header.""" - internal_assert(which in allowed_headers, "invalid header type", which) + internal_assert( + which.startswith("package") or which in ( + "none", "initial", "__coconut__", "sys", "code", "file", + ), + "invalid header type", + which, + ) if which == "none": return "" @@ -279,9 +282,13 @@ def getheader(which, target="", use_hash=None, no_tco=False, strict=False): elif target_info >= (3, 5): header += "from __future__ import generator_stop\n" - if which == "package": + if which.startswith("package"): + levels_up = int(which[len("package:"):]) + coconut_file_path = "_coconut_os_path.dirname(_coconut_os_path.abspath(__file__))" + for _ in range(levels_up): + coconut_file_path = "_coconut_os_path.dirname(" + coconut_file_path + ")" return header + '''import sys as _coconut_sys, os.path as _coconut_os_path -_coconut_file_path = _coconut_os_path.dirname(_coconut_os_path.abspath(__file__)) +_coconut_file_path = {coconut_file_path} _coconut_cached_module = _coconut_sys.modules.get({__coconut__}) if _coconut_cached_module is not None and _coconut_os_path.dirname(_coconut_cached_module.__file__) != _coconut_file_path: del _coconut_sys.modules[{__coconut__}] @@ -290,7 +297,7 @@ def getheader(which, target="", use_hash=None, no_tco=False, strict=False): from __coconut__ import * _coconut_sys.path.pop(0) -'''.format(**format_dict) + section("Compiled Coconut") +'''.format(coconut_file_path=coconut_file_path, **format_dict) + section("Compiled Coconut") if which == "sys": return header + '''import sys as _coconut_sys diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index fcef4f520..4fe848857 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -1,5 +1,5 @@ class _coconut{object}:{comment.everything_here_must_be_copied_to_stub_file} - import collections, copy, functools, types, itertools, operator, types, weakref, threading + import collections, copy, functools, types, itertools, operator, weakref, threading {bind_lru_cache}{import_asyncio}{import_pickle} {import_OrderedDict} {import_collections_abc} diff --git a/coconut/root.py b/coconut/root.py index a6f4a809c..d26b3201e 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 8 +DEVELOP = 9 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 0a498af0ee9d989be1765c25ebcd9ab5e92abe28 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 24 Feb 2019 00:50:15 -0800 Subject: [PATCH 049/163] Fix testing on diff py versions --- tests/main_test.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/main_test.py b/tests/main_test.py index 3223ff330..83353f0cc 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -130,13 +130,18 @@ def call(cmd, assert_output=False, check_mypy=False, check_errors=True, stderr_f assert any(x in last_line for x in assert_output), "Expected " + ", ".join(assert_output) + "; got " + repr(last_line) +def call_python(args, **kwargs): + """Calls the current Python.""" + call([sys.executable] + args, **kwargs) + + def call_coconut(args, **kwargs): """Calls Coconut.""" if "--jobs" not in args and not PYPY and not PY26: args = ["--jobs", "sys"] + args if "--mypy" in args and "check_mypy" not in kwargs: kwargs["check_mypy"] = True - call([sys.executable, "-m", "coconut"] + args, **kwargs) + call_python(["-m", "coconut"] + args, **kwargs) def comp(path=None, folder=None, file=None, args=[], **kwargs): @@ -232,12 +237,12 @@ def comp_sys(args=[], **kwargs): def run_src(**kwargs): """Runs runner.py.""" - call(["python", os.path.join(dest, "runner.py")], assert_output=True, **kwargs) + call_python([os.path.join(dest, "runner.py")], assert_output=True, **kwargs) def run_extras(**kwargs): """Runs extras.py.""" - call(["python", os.path.join(dest, "extras.py")], assert_output=True, check_errors=False, stderr_first=True, **kwargs) + call_python([os.path.join(dest, "extras.py")], assert_output=True, check_errors=False, stderr_first=True, **kwargs) def run(args=[], agnostic_target=None, use_run_arg=False, expect_retcode=0): @@ -279,7 +284,7 @@ def comp_pyston(args=[], **kwargs): def run_pyston(**kwargs): """Runs pyston.""" - call(["python", os.path.join(pyston, "runner.py")], assert_output=True, **kwargs) + call_python([os.path.join(pyston, "runner.py")], assert_output=True, **kwargs) def comp_pyprover(args=[], **kwargs): @@ -292,7 +297,7 @@ def comp_pyprover(args=[], **kwargs): def run_pyprover(**kwargs): """Runs pyprover.""" call(["pip", "install", "-e", pyprover]) - call(["python", os.path.join(pyprover, "pyprover", "tests.py")], assert_output=True, **kwargs) + call_python([os.path.join(pyprover, "pyprover", "tests.py")], assert_output=True, **kwargs) def comp_prelude(args=[], **kwargs): @@ -307,7 +312,7 @@ def comp_prelude(args=[], **kwargs): def run_prelude(**kwargs): """Runs coconut-prelude.""" call(["pip", "install", "-e", prelude]) - call(["python", os.path.join(prelude, "prelude", "prelude_test.py")], assert_output=True, **kwargs) + call_python([os.path.join(prelude, "prelude", "prelude_test.py")], assert_output=True, **kwargs) def comp_all(args=[], **kwargs): @@ -343,7 +348,7 @@ def test_pipe(self): call('echo ' + escape(coconut_snip) + "| coconut -s", shell=True, assert_output=True) def test_convenience(self): - call(["python", "-c", 'from coconut.convenience import parse; exec(parse("' + coconut_snip + '"))'], assert_output=True) + call_python(["-c", 'from coconut.convenience import parse; exec(parse("' + coconut_snip + '"))'], assert_output=True) def test_import_hook(self): sys.path.append(src) @@ -368,7 +373,7 @@ def test_runnable_nowrite(self): def test_compile_to_file(self): with remove_when_done(runnable_py): call_coconut([runnable_coco, runnable_py]) - call(["python", runnable_py, "--arg"], assert_output=True) + call_python([runnable_py, "--arg"], assert_output=True) if IPY: def test_ipython_extension(self): From 7eb47f42f36c38db639492eb02ff3b281c664ad3 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 24 Feb 2019 02:09:59 -0800 Subject: [PATCH 050/163] Fix pickling on Python 2 --- coconut/compiler/header.py | 28 +++++++++++++++++++--------- coconut/root.py | 2 +- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 6dc70958a..694b23841 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -192,11 +192,6 @@ def pattern_prepender(func): return _coconut.functools.partial(makedata, data_type) ''' if not strict else "" ), - __coconut__=( - '"__coconut__"' if target_startswith == "3" - else 'b"__coconut__"' if target_startswith == "2" - else 'str("__coconut__")' - ), ) format_dict["import_typing_NamedTuple"] = _indent( @@ -292,12 +287,27 @@ def getheader(which, target="", use_hash=None, no_tco=False, strict=False): _coconut_cached_module = _coconut_sys.modules.get({__coconut__}) if _coconut_cached_module is not None and _coconut_os_path.dirname(_coconut_cached_module.__file__) != _coconut_file_path: del _coconut_sys.modules[{__coconut__}] -_coconut_sys.path.insert(0, _coconut_file_path) +_coconut_sys.path.append(_coconut_file_path) from __coconut__ import {underscore_imports} from __coconut__ import * -_coconut_sys.path.pop(0) - -'''.format(coconut_file_path=coconut_file_path, **format_dict) + section("Compiled Coconut") +{sys_path_pop} + +'''.format( + coconut_file_path=coconut_file_path, + __coconut__=( + '"__coconut__"' if target_startswith == "3" + else 'b"__coconut__"' if target_startswith == "2" + else 'str("__coconut__")' + ), + sys_path_pop=( + # we can't pop on Python 2 if we want __coconut__ objects to be pickleable + "_coconut_sys.path.pop()" if target_startswith == "3" + else "" if target_startswith == "2" + else '''if _coconut_sys.version_info >= (3,): + _coconut_sys.path.pop()''' + ), + **format_dict + ) + section("Compiled Coconut") if which == "sys": return header + '''import sys as _coconut_sys diff --git a/coconut/root.py b/coconut/root.py index d26b3201e..b40223836 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 9 +DEVELOP = 10 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 34a24d349beeaedd035a33dbd7b076159748a695 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 24 Feb 2019 02:23:35 -0800 Subject: [PATCH 051/163] Fix hash generation --- coconut/command/command.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index 338d7f993..6bf4a796b 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -361,6 +361,7 @@ def compile(self, codepath, destpath=None, package=False, run=False, force=False with openfile(codepath, "r") as opened: code = readfile(opened) + package_level = -1 if destpath is not None: destpath = fixpath(destpath) destdir = os.path.dirname(destpath) @@ -370,8 +371,6 @@ def compile(self, codepath, destpath=None, package=False, run=False, force=False package_level = self.get_package_level(codepath) if package_level == 0: self.create_package(destdir) - else: - package_level = -1 foundhash = None if force else self.has_hash_of(destpath, code, package_level) if foundhash: From 0c4c1c7abd3eaaff17f7adbdfd342777b63c500f Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 24 Feb 2019 02:28:52 -0800 Subject: [PATCH 052/163] Test py 3.7 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c09e349e3..30298f86d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ python: - '3.4' - '3.5' - '3.6' +- '3.7' - pypy3 install: - make install From b3dce78eea09c4572ecdd44e1a532452d9d39f3a Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 24 Feb 2019 13:46:41 -0800 Subject: [PATCH 053/163] Fix py 2.6, 3.7 testing --- .travis.yml | 1 - tests/main_test.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 30298f86d..c09e349e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,6 @@ python: - '3.4' - '3.5' - '3.6' -- '3.7' - pypy3 install: - make install diff --git a/tests/main_test.py b/tests/main_test.py index 83353f0cc..eaddd0b41 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -107,10 +107,10 @@ def call(cmd, assert_output=False, check_mypy=False, check_errors=True, stderr_f out = "".join(out) lines = out.splitlines() if expect_retcode is not None: - assert retcode == expect_retcode, "Return code not as expected ({} != {}) in: {!r}".format( - retcode, - expect_retcode, - cmd, + assert retcode == expect_retcode, "Return code not as expected ({retcode} != {expect_retcode}) in: {cmd!r}".format( + retcode=retcode, + expect_retcode=expect_retcode, + cmd=cmd, ) for line in lines: assert "CoconutInternalException" not in line, "CoconutInternalException in " + repr(line) From a134351b3a5627d7218fd3423e6d4f227e7380ce Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 24 Feb 2019 17:34:57 -0800 Subject: [PATCH 054/163] Fix py26 testing --- tests/main_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/main_test.py b/tests/main_test.py index eaddd0b41..65dd59816 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -141,7 +141,10 @@ def call_coconut(args, **kwargs): args = ["--jobs", "sys"] + args if "--mypy" in args and "check_mypy" not in kwargs: kwargs["check_mypy"] = True - call_python(["-m", "coconut"] + args, **kwargs) + if PY26: + call(["coconut"] + args, **kwargs) + else: + call_python(["-m", "coconut"] + args, **kwargs) def comp(path=None, folder=None, file=None, args=[], **kwargs): From 008060bc9b3d4b70b9dedea5a4e6a480929b7f60 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 25 Feb 2019 22:55:13 -0800 Subject: [PATCH 055/163] Fix package header --- coconut/compiler/header.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 694b23841..ccd030f35 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -287,7 +287,7 @@ def getheader(which, target="", use_hash=None, no_tco=False, strict=False): _coconut_cached_module = _coconut_sys.modules.get({__coconut__}) if _coconut_cached_module is not None and _coconut_os_path.dirname(_coconut_cached_module.__file__) != _coconut_file_path: del _coconut_sys.modules[{__coconut__}] -_coconut_sys.path.append(_coconut_file_path) +_coconut_sys.path.insert(0, _coconut_file_path) from __coconut__ import {underscore_imports} from __coconut__ import * {sys_path_pop} @@ -301,10 +301,10 @@ def getheader(which, target="", use_hash=None, no_tco=False, strict=False): ), sys_path_pop=( # we can't pop on Python 2 if we want __coconut__ objects to be pickleable - "_coconut_sys.path.pop()" if target_startswith == "3" + "_coconut_sys.path.pop(0)" if target_startswith == "3" else "" if target_startswith == "2" else '''if _coconut_sys.version_info >= (3,): - _coconut_sys.path.pop()''' + _coconut_sys.path.pop(0)''' ), **format_dict ) + section("Compiled Coconut") From 61f42e09f2da0fb6ab55d8634caaa75efd452518 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 25 Feb 2019 23:10:16 -0800 Subject: [PATCH 056/163] Rename make check --- CONTRIBUTING.md | 2 +- Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 06b45d880..4f501f3a0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -156,7 +156,7 @@ After you've tested your changes locally, you'll want to add more permanent test ## Release Process 1. Preparation: - 1. Run `make check` and update dependencies as necessary + 1. Run `make check-reqs` and update dependencies as necessary 1. Run `make format` 1. Check changes in [`compiled-cocotest`](https://github.com/evhub/compiled-cocotest) and [`pyprover`](https://github.com/evhub/pyprover) 1. Check [Codacy issues](https://www.codacy.com/app/evanjhub) (for `coconut` and `compiled-cocotest`) and [LGTM alerts](https://lgtm.com/projects/g/evhub/coconut/) diff --git a/Makefile b/Makefile index 8feac8946..c48e79791 100644 --- a/Makefile +++ b/Makefile @@ -82,8 +82,8 @@ just-upload: .PHONY: upload upload: clean dev just-upload -.PHONY: check -check: +.PHONY: check-reqs +check-reqs: python ./coconut/requirements.py .PHONY: profile-code From 6f67f0e7b51c690b946cc328c999b9328cb9862c Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 26 Feb 2019 01:55:24 -0800 Subject: [PATCH 057/163] Fix tests --- coconut/command/command.py | 5 +++-- tests/main_test.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index 6bf4a796b..43af3f914 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -619,10 +619,11 @@ def run_mypy(self, paths=(), code=None): if code is not None: args += ["-c", code] for line, is_err in mypy_run(args): - if code is None or line not in self.mypy_errs: - printerr(line) if line not in self.mypy_errs: + printerr(line) self.mypy_errs.append(line) + elif code is None: + printerr(line) self.register_error(errmsg="MyPy error") def start_jupyter(self, args): diff --git a/tests/main_test.py b/tests/main_test.py index 65dd59816..dbfbe8c9f 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -66,7 +66,7 @@ coconut_snip = r"msg = ''; pmsg = print$(msg); `pmsg`" mypy_snip = r"a: str = count()[0]" -mypy_snip_err = 'error: Incompatible types in assignment (expression has type "int", variable has type "unicode")' +mypy_snip_err = 'error: Incompatible types in assignment (expression has type "int", variable has type "str")' mypy_args = ["--follow-imports", "silent", "--ignore-missing-imports"] From 49bd10d586424f4600558671e5896b8e36e616f1 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 26 Feb 2019 11:47:06 -0800 Subject: [PATCH 058/163] Fix prelude test --- coconut/constants.py | 1 + tests/main_test.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index a4abcc14f..b791fff9e 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -83,6 +83,7 @@ def checksum(data): PY33 = sys.version_info >= (3, 3) PY34 = sys.version_info >= (3, 4) PY35 = sys.version_info >= (3, 5) +PY36 = sys.version_info >= (3, 6) IPY = (PY2 and not PY26) or PY34 # ----------------------------------------------------------------------------------------------------------------------- diff --git a/tests/main_test.py b/tests/main_test.py index dbfbe8c9f..a32749840 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -35,6 +35,7 @@ IPY, PY34, PY35, + PY36, icoconut_kernel_names, ) @@ -307,8 +308,8 @@ def comp_prelude(args=[], **kwargs): """Compiles evhub/coconut-prelude.""" call(["git", "clone", prelude_git]) call_coconut([os.path.join(prelude, "setup.coco"), "--strict"] + args, **kwargs) - if MYPY: - args.append("--mypy") + if PY36: + args.append("--target", "3.6", "--mypy") call_coconut([os.path.join(prelude, "prelude-source"), os.path.join(prelude, "prelude"), "--strict"] + args, **kwargs) From dbe3ec0f17dbde1fdabe285789f13ecf74257da2 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 26 Feb 2019 15:34:47 -0800 Subject: [PATCH 059/163] Fix kernel test --- tests/src/extras.coco | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/src/extras.coco b/tests/src/extras.coco index afec1ccdf..0bffab8b7 100644 --- a/tests/src/extras.coco +++ b/tests/src/extras.coco @@ -3,7 +3,11 @@ import os from coconut.__coconut__ import consume as coc_consume # type: ignore -from coconut.constants import IPY, WINDOWS # type: ignore +from coconut.constants import ( + IPY, + PY35, + WINDOWS, +) # type: ignore from coconut.exceptions import ( CoconutSyntaxError, CoconutStyleError, @@ -19,6 +23,8 @@ from coconut.convenience import ( ) if IPY and not WINDOWS: + if PY35: + import asyncio from coconut.icoconut import CoconutKernel # type: ignore else: CoconutKernel = None # type: ignore @@ -114,6 +120,8 @@ def main(): assert parse("f''") assert parse("def f(*, x=None) = x") if CoconutKernel is not None: + if PY35: + asyncio.set_event_loop(asyncio.new_event_loop()) k = CoconutKernel() exec_result = k.do_execute("derp = pow$(?, 2)", False, True, {"two": "(+)(1, 1)"}, True) |> unwrap_future assert exec_result["status"] == "ok" From b1daa4c5bc5eefeac04ed33da7c149dc62d65b11 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 1 Mar 2019 18:58:42 -0800 Subject: [PATCH 060/163] Make format strings universal Resolves #301. --- DOCS.md | 9 ++- coconut/compiler/compiler.py | 79 ++++++++++++++++++++++++--- coconut/compiler/grammar.py | 2 +- coconut/constants.py | 1 + coconut/convenience.py | 2 +- coconut/root.py | 2 +- tests/src/cocotest/agnostic/main.coco | 7 +++ tests/src/extras.coco | 18 +++--- 8 files changed, 94 insertions(+), 26 deletions(-) diff --git a/DOCS.md b/DOCS.md index d8ef60a71..ee67b6028 100644 --- a/DOCS.md +++ b/DOCS.md @@ -200,9 +200,8 @@ Finally, while Coconut will try to compile Python-3-specific syntax to its unive - keyword-only function arguments (use pattern-matching function definition instead), - destructuring assignment with `*`s (use pattern-matching instead), - tuples and lists with `*` unpacking or dicts with `**` unpacking (requires `--target 3.5`), -- `@` as matrix multiplication (requires `--target 3.5`), -- `async` and `await` statements (requires `--target 3.5`), and -- formatting strings by prefixing them with `f` (requires `--target 3.6`). +- `@` as matrix multiplication (requires `--target 3.5`), and +- `async` and `await` statements (requires `--target 3.5`). ### Allowable Targets @@ -2374,8 +2373,8 @@ Each _mode_ has two components: what parser it uses, and what header it prepends + parser: eval * Can only parse a Coconut expression, not a statement. + header: none -- `"debug"`: - + parser: debug +- `"any"`: + + parser: any * Can parse any Coconut code, allows leading whitespace, and has no trailing newline. + header: none diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 4b2a64605..dad09629b 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -73,6 +73,7 @@ case_check_var, function_match_error_var, legal_indent_chars, + format_var, ) from coconut.exceptions import ( CoconutException, @@ -421,6 +422,7 @@ def bind(self): self.await_item <<= trace(attach(self.await_item_ref, self.await_item_handle)) self.ellipsis <<= trace(attach(self.ellipsis_ref, self.ellipsis_handle)) self.case_stmt <<= trace(attach(self.case_stmt_ref, self.case_stmt_handle)) + self.f_string <<= attach(self.f_string_ref, self.f_string_handle) self.decoratable_normal_funcdef_stmt <<= trace(attach( self.decoratable_normal_funcdef_stmt_ref, @@ -443,7 +445,6 @@ def bind(self): self.endline_semicolon <<= attach(self.endline_semicolon_ref, self.endline_semicolon_check) self.async_stmt <<= attach(self.async_stmt_ref, self.async_stmt_check) self.async_comp_for <<= attach(self.async_comp_for_ref, self.async_comp_check) - self.f_string <<= attach(self.f_string_ref, self.f_string_check) def copy_skips(self): """Copy the line skips.""" @@ -1899,6 +1900,72 @@ def case_stmt_handle(self, loc, tokens): out += "if not " + check_var + default return out + def f_string_handle(self, original, loc, tokens): + """Handle Python 3.6 format strings.""" + internal_assert(len(tokens) == 1, "invalid format string tokens", tokens) + string = tokens[0] + if self.target_info >= (3, 6): + return "f" + string + else: + # strip raw r + raw = string.startswith("r") + if raw: + string = string[1:] + + # strip wrappers + internal_assert(string.startswith(strwrapper) and string.endswith(unwrapper)) + string = string[1:-1] + + # get text + old_text, strchar = self.get_ref("str", string) + + # separate expressions + new_text = "" + exprs = [] + saw_brace = False + in_expr = False + expr_level = 0 + for c in old_text: + if saw_brace: + saw_brace = False + if c == "{": + new_text += c + elif c == "}": + raise self.make_err(CoconutSyntaxError, "empty expressing in format string", original, loc) + else: + in_expr = True + expr_level = paren_change(c) + exprs.append(c) + elif in_expr: + if expr_level < 0: + expr_level += paren_change(c) + exprs[-1] += c + elif expr_level > 0: + raise self.make_err(CoconutSyntaxError, "imbalanced parentheses in format string expression", original, loc) + elif c in "!:}": # these characters end the expr + in_expr = False + name = format_var + "_" + str(len(exprs) - 1) + new_text += name + c + else: + exprs[-1] += c + elif c == "{": + saw_brace = True + new_text += c + else: + new_text += c + + # handle dangling detections + if saw_brace: + raise self.make_err(CoconutSyntaxError, "format string ends with unescaped brace (escape by doubling to '{{')", original, loc) + if in_expr: + raise self.make_err(CoconutSyntaxError, "imbalanced braces in format string (escape braces by doubling to '{{' and '}}')", original, loc) + + # generate format call + return ("r" if raw else "") + strchar + new_text + strchar + ".format(" + ", ".join( + format_var + "_" + str(i) + "=" + expr + for i, expr in enumerate(exprs) + ) + ")" + # end: COMPILER HANDLERS # ----------------------------------------------------------------------------------------------------------------------- # CHECKING HANDLERS: @@ -1972,10 +2039,6 @@ def async_comp_check(self, original, loc, tokens): """Check for Python 3.6 async comprehension.""" return self.check_py("36", "async comprehension", original, loc, tokens) - def f_string_check(self, original, loc, tokens): - """Handle Python 3.6 format strings.""" - return self.check_py("36", "format string", original, loc, tokens) - # end: CHECKING HANDLERS # ----------------------------------------------------------------------------------------------------------------------- # ENDPOINTS: @@ -2018,13 +2081,13 @@ def parse_eval(self, inputstring): """Parse eval code.""" return self.parse(inputstring, self.eval_parser, {"strip": True}, {"header": "none", "initial": "none"}) - def parse_debug(self, inputstring): - """Parse debug code.""" + def parse_any(self, inputstring): + """Parse any code.""" return self.parse(inputstring, self.file_parser, {"strip": True}, {"header": "none", "initial": "none", "final_endline": False}) def warm_up(self): """Warm up the compiler by running something through it.""" - result = self.parse_debug("") + result = self.parse_any("") internal_assert(result == "", "compiler warm-up should produce no code; instead got", result) # end: ENDPOINTS diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 6b182836a..557f447a3 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -826,7 +826,7 @@ class Grammar(object): b_string = Combine((bit_b + raw_r | raw_r + bit_b) + string_item) unicode_u = CaselessLiteral("u").suppress() u_string_ref = Combine((unicode_u + raw_r | raw_r + unicode_u) + string_item) - format_f = CaselessLiteral("f") + format_f = CaselessLiteral("f").suppress() f_string_ref = Combine((format_f + raw_r | raw_r + format_f) + string_item) string = trace(b_string | u_string | f_string) moduledoc = string + newline diff --git a/coconut/constants.py b/coconut/constants.py index b791fff9e..35e734354 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -381,6 +381,7 @@ def checksum(data): tre_check_var = reserved_prefix + "_is_recursive" none_coalesce_var = reserved_prefix + "_none_coalesce_item" func_var = reserved_prefix + "_func" +format_var = reserved_prefix + "_format" # prefer Matcher.get_temp_var to proliferating more match vars here match_to_var = reserved_prefix + "_match_to" diff --git a/coconut/convenience.py b/coconut/convenience.py index 86e664afa..cbd6a5aa4 100644 --- a/coconut/convenience.py +++ b/coconut/convenience.py @@ -80,7 +80,7 @@ def version(which="num"): "block": lambda comp: comp.parse_block, "single": lambda comp: comp.parse_single, "eval": lambda comp: comp.parse_eval, - "debug": lambda comp: comp.parse_debug, + "any": lambda comp: comp.parse_any, } diff --git a/coconut/root.py b/coconut/root.py index b40223836..596c5af8f 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 10 +DEVELOP = 11 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index b67ecad1d..64fef454d 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -500,6 +500,13 @@ def main_test(): assert False assert 4 == range(2,20) |> filter$(i-> i > 3) |> .$[0] from importlib import reload + x = 1 + y = "2" + assert f"{x} == {y}" == "1 == 2" + assert f"{x!r} == {y!r}" == "1 == '2'" + assert f"{({})}" == "{}" == f"{({})!r}" + assert f"{{" == "{" + assert f"}}" == "}" return True def tco_func() = tco_func() diff --git a/tests/src/extras.coco b/tests/src/extras.coco index 0bffab8b7..2237e89ed 100644 --- a/tests/src/extras.coco +++ b/tests/src/extras.coco @@ -67,10 +67,10 @@ def main(): assert parse("abc", "package") assert parse("abc", "block") == "abc\n" == parse("abc", "single") assert parse("abc", "eval") == "abc\n" == parse(" abc", "eval") - assert parse("abc", "debug") == "abc" - assert parse("x |> map$(f)", "debug") == "map(f, x)" + assert parse("abc", "any") == "abc" + assert parse("x |> map$(f)", "any") == "map(f, x)" assert "_coconut" not in parse("a |> .b |> .m() |> f$(x) |> .[0]", "block") - assert parse("abc # derp", "debug") == "abc # derp" + assert parse("abc # derp", "any") == "abc # derp" assert_raises(-> parse(" abc", "file"), CoconutException) assert_raises(-> parse("'"), CoconutException) assert_raises(-> parse("("), CoconutException) @@ -83,13 +83,13 @@ def main(): assert parse("u''") assert parse("def f(x):\\\n pass") assert parse("abc ") - assert parse("abc # derp", "debug") == "abc # derp" + assert parse("abc # derp", "any") == "abc # derp" setup(line_numbers=True) - assert parse("abc", "debug") == "abc # line 1" + assert parse("abc", "any") == "abc # line 1" setup(keep_lines=True) - assert parse("abc", "debug") == "abc # abc" + assert parse("abc", "any") == "abc # abc" setup(line_numbers=True, keep_lines=True) - assert parse("abc", "debug") == "abc # line 1: abc" + assert parse("abc", "any") == "abc # line 1: abc" setup() assert "prepattern" in parse("\n", mode="file") assert "datamaker" in parse("\n", mode="file") @@ -108,16 +108,14 @@ def main(): assert_raises(-> cmd("-f"), SystemExit) assert_raises(-> cmd("-pa ."), SystemExit) assert_raises(-> cmd("-n . ."), SystemExit) - assert_raises(-> parse("f''"), CoconutException) assert_raises(-> parse("f$()"), CoconutSyntaxError) assert_raises(-> parse("f(*x, y)"), CoconutSyntaxError) assert_raises(-> parse("def f(x) = return x"), CoconutSyntaxError) assert_raises(-> parse("def f(x) =\n return x"), CoconutSyntaxError) setup(target="2.7") - assert parse("from io import BytesIO", mode="debug") == "from io import BytesIO" + assert parse("from io import BytesIO", mode="any") == "from io import BytesIO" assert_raises(-> parse("def f(*, x=None) = x"), CoconutTargetError) setup(target="3.6") - assert parse("f''") assert parse("def f(*, x=None) = x") if CoconutKernel is not None: if PY35: From fc1706ef8729176ef302bdcf761531cb7ae93d2d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 1 Mar 2019 19:02:50 -0800 Subject: [PATCH 061/163] Fix fstring exprs --- coconut/compiler/compiler.py | 2 +- coconut/root.py | 2 +- tests/src/cocotest/agnostic/main.coco | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index dad09629b..f82874917 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -1962,7 +1962,7 @@ def f_string_handle(self, original, loc, tokens): # generate format call return ("r" if raw else "") + strchar + new_text + strchar + ".format(" + ", ".join( - format_var + "_" + str(i) + "=" + expr + format_var + "_" + str(i) + "=(" + expr + ")" for i, expr in enumerate(exprs) ) + ")" diff --git a/coconut/root.py b/coconut/root.py index 596c5af8f..37bb0762a 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 11 +DEVELOP = 12 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index 64fef454d..8c45a5bb9 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -507,6 +507,7 @@ def main_test(): assert f"{({})}" == "{}" == f"{({})!r}" assert f"{{" == "{" assert f"}}" == "}" + assert f"{1, 2}" == "(1, 2)" return True def tco_func() = tco_func() From d39d35fb41a9a8147094e18b9b2509775c74bdd0 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 1 Mar 2019 19:07:26 -0800 Subject: [PATCH 062/163] Improve eval parser --- coconut/compiler/compiler.py | 2 +- tests/src/extras.coco | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index f82874917..8ef879da1 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -2079,7 +2079,7 @@ def parse_sys(self, inputstring): def parse_eval(self, inputstring): """Parse eval code.""" - return self.parse(inputstring, self.eval_parser, {"strip": True}, {"header": "none", "initial": "none"}) + return self.parse(inputstring, self.eval_parser, {"strip": True}, {"header": "none", "initial": "none", "final_endline": False}) def parse_any(self, inputstring): """Parse any code.""" diff --git a/tests/src/extras.coco b/tests/src/extras.coco index 2237e89ed..728751d17 100644 --- a/tests/src/extras.coco +++ b/tests/src/extras.coco @@ -66,7 +66,7 @@ def main(): assert parse("abc", "file") assert parse("abc", "package") assert parse("abc", "block") == "abc\n" == parse("abc", "single") - assert parse("abc", "eval") == "abc\n" == parse(" abc", "eval") + assert parse("abc", "eval") == "abc" == parse(" abc", "eval") assert parse("abc", "any") == "abc" assert parse("x |> map$(f)", "any") == "map(f, x)" assert "_coconut" not in parse("a |> .b |> .m() |> f$(x) |> .[0]", "block") From 9ed2e9a99b64b73b6dfe4bd71c6e97ee7ec56ebb Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 1 Mar 2019 19:51:25 -0800 Subject: [PATCH 063/163] Support assignment exprs Resolves #482. --- coconut/compiler/compiler.py | 11 ++++++++--- coconut/compiler/grammar.py | 18 ++++++++++++++---- coconut/constants.py | 3 ++- coconut/root.py | 2 +- tests/src/extras.coco | 9 +++++++-- 5 files changed, 32 insertions(+), 11 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 8ef879da1..17c477bdf 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -445,6 +445,7 @@ def bind(self): self.endline_semicolon <<= attach(self.endline_semicolon_ref, self.endline_semicolon_check) self.async_stmt <<= attach(self.async_stmt_ref, self.async_stmt_check) self.async_comp_for <<= attach(self.async_comp_for_ref, self.async_comp_check) + self.namedexpr <<= attach(self.namedexpr_ref, self.namedexpr_check) def copy_skips(self): """Copy the line skips.""" @@ -2017,15 +2018,15 @@ def nonlocal_check(self, original, loc, tokens): def star_assign_item_check(self, original, loc, tokens): """Check for Python 3 starred assignment.""" - return self.check_py("3", "starred assignment (add 'match' to front to produce universal code)", original, loc, tokens) + return self.check_py("3", "starred assignment (use 'match' to produce universal code)", original, loc, tokens) def star_expr_check(self, original, loc, tokens): """Check for Python 3.5 star unpacking.""" - return self.check_py("35", "star unpacking (add 'match' to front to produce universal code)", original, loc, tokens) + return self.check_py("35", "star unpacking (use 'match' to produce universal code)", original, loc, tokens) def star_sep_check(self, original, loc, tokens): """Check for Python 3 keyword-only arguments.""" - return self.check_py("3", "keyword-only argument separator (add 'match' to front to produce universal code)", original, loc, tokens) + return self.check_py("3", "keyword-only argument separator (use 'match' to produce universal code)", original, loc, tokens) def matrix_at_check(self, original, loc, tokens): """Check for Python 3.5 matrix multiplication.""" @@ -2039,6 +2040,10 @@ def async_comp_check(self, original, loc, tokens): """Check for Python 3.6 async comprehension.""" return self.check_py("36", "async comprehension", original, loc, tokens) + def namedexpr_check(self, original, loc, tokens): + """Check for Python 3.8 assignment expressions.""" + return self.check_py("38", "assignment expression", original, loc, tokens) + # end: CHECKING HANDLERS # ----------------------------------------------------------------------------------------------------------------------- # ENDPOINTS: diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 557f447a3..05c1a8bfe 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -699,6 +699,7 @@ class Grammar(object): unsafe_dubcolon = Literal("::") unsafe_colon = Literal(":") colon = ~unsafe_dubcolon + unsafe_colon + colon_eq = Literal(":=") semicolon = Literal(";") eq = Literal("==") equals = ~eq + Literal("=") @@ -764,6 +765,7 @@ class Grammar(object): matrix_at = Forward() test = Forward() + namedexpr_test = Forward() test_no_chain, dubcolon = disable_inside(test, unsafe_dubcolon) test_no_infix, backtick = disable_inside(test, unsafe_backtick) @@ -875,6 +877,7 @@ class Grammar(object): testlist = trace(itemlist(test, comma, suppress_trailing=False)) testlist_star_expr = trace(itemlist(test | star_expr, comma, suppress_trailing=False)) + testlist_star_namedexpr = trace(itemlist(namedexpr_test | star_expr, comma, suppress_trailing=False)) testlist_has_comma = trace(addspace(OneOrMore(condense(test + comma)) + Optional(test))) yield_from = Forward() @@ -1015,7 +1018,7 @@ class Grammar(object): subscriptgroup = attach(slicetestgroup + sliceopgroup + Optional(sliceopgroup) | test, subscriptgroup_handle) subscriptgrouplist = itemlist(subscriptgroup, comma) - testlist_comp = addspace((test | star_expr) + comp_for) | testlist_star_expr + testlist_comp = addspace((namedexpr_test | star_expr) + comp_for) | testlist_star_namedexpr list_comp = condense(lbrack + Optional(testlist_comp) + rbrack) paren_atom = condense(lparen + Optional(yield_expr | testlist_comp) + rparen) op_atom = lparen.suppress() + op_item + rparen.suppress() @@ -1273,6 +1276,13 @@ class Grammar(object): ) test_no_cond <<= trace(lambdef_no_cond | test_item) + namedexpr = Forward() + namedexpr_ref = addspace(name + colon_eq + test) + namedexpr_test <<= trace( + test + ~colon_eq + | namedexpr, + ) + async_comp_for = Forward() classlist_ref = Optional( lparen.suppress() + rparen.suppress() @@ -1409,11 +1419,11 @@ class Grammar(object): exec_stmt = Forward() assert_stmt = addspace(keyword("assert") - testlist) if_stmt = condense( - addspace(keyword("if") - condense(test - suite)) - - ZeroOrMore(addspace(keyword("elif") - condense(test - suite))) + addspace(keyword("if") - condense(namedexpr_test - suite)) + - ZeroOrMore(addspace(keyword("elif") - condense(namedexpr_test - suite))) - Optional(else_stmt), ) - while_stmt = addspace(keyword("while") - condense(test - suite - Optional(else_stmt))) + while_stmt = addspace(keyword("while") - condense(namedexpr_test - suite - Optional(else_stmt))) for_stmt = addspace(keyword("for") - assignlist - keyword("in") - condense(testlist - suite - Optional(else_stmt))) except_clause = attach( keyword("except").suppress() + ( diff --git a/coconut/constants.py b/coconut/constants.py index 35e734354..40221cf6b 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -336,12 +336,13 @@ def checksum(data): py2_vers = ((2, 6), (2, 7)) py3_vers = ((3, 2), (3, 3), (3, 4), (3, 5), (3, 6), (3, 7)) -specific_targets = ("2", "27", "3", "33", "35", "36") +specific_targets = ("2", "27", "3", "33", "35", "36", "38") pseudo_targets = { "universal": "", "26": "2", "32": "3", "34": "33", + "37": "36", } targets = ("",) + specific_targets diff --git a/coconut/root.py b/coconut/root.py index 37bb0762a..60f3f9cd3 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 12 +DEVELOP = 13 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/extras.coco b/tests/src/extras.coco index 728751d17..5f31450f8 100644 --- a/tests/src/extras.coco +++ b/tests/src/extras.coco @@ -13,6 +13,7 @@ from coconut.exceptions import ( CoconutStyleError, CoconutSyntaxError, CoconutTargetError, + CoconutParseError, ) # type: ignore from coconut.convenience import ( CoconutException, @@ -50,7 +51,7 @@ def unwrap_future(maybe_future): return maybe_future.result() return maybe_future -def main(): +def test_extras(): if IPY: import coconut.highlighter # type: ignore assert consume(range(10), keep_last=1)[0] == 9 == coc_consume(range(10), keep_last=1)[0] @@ -112,11 +113,15 @@ def main(): assert_raises(-> parse("f(*x, y)"), CoconutSyntaxError) assert_raises(-> parse("def f(x) = return x"), CoconutSyntaxError) assert_raises(-> parse("def f(x) =\n return x"), CoconutSyntaxError) + assert_raises(-> parse("a := b"), CoconutParseError) + assert_raises(-> parse("(a := b)"), CoconutTargetError) setup(target="2.7") assert parse("from io import BytesIO", mode="any") == "from io import BytesIO" assert_raises(-> parse("def f(*, x=None) = x"), CoconutTargetError) setup(target="3.6") assert parse("def f(*, x=None) = x") + setup(target="3.8") + assert parse("(a := b)") if CoconutKernel is not None: if PY35: asyncio.set_event_loop(asyncio.new_event_loop()) @@ -149,4 +154,4 @@ def main(): if __name__ == "__main__": print("Expect Coconut errors below from running extras:") print("(but make sure you get a after them)") - main() + test_extras() From b47e4e052368af83be81861897817c5f5e8c5fc1 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 1 Mar 2019 20:02:23 -0800 Subject: [PATCH 064/163] Fix or pattern Resolves #464. --- coconut/compiler/matching.py | 9 ++++++--- coconut/root.py | 2 +- tests/src/cocotest/agnostic/main.coco | 2 ++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/coconut/compiler/matching.py b/coconut/compiler/matching.py index 5ede8508c..b556b717b 100644 --- a/coconut/compiler/matching.py +++ b/coconut/compiler/matching.py @@ -115,9 +115,12 @@ def __init__(self, loc, check_var, checkdefs=None, names=None, var_index=0): self.others = [] self.guards = [] - def duplicate(self): + def duplicate(self, separate_names=False): """Duplicates the matcher to others.""" - other = Matcher(self.loc, self.check_var, self.checkdefs, self.names, self.var_index) + new_names = self.names + if separate_names: + new_names = new_names.copy() + other = Matcher(self.loc, self.check_var, self.checkdefs, new_names, self.var_index) other.insert_check(0, "not " + self.check_var) self.others.append(other) return other @@ -604,7 +607,7 @@ def match_and(self, tokens, item): def match_or(self, tokens, item): """Matches or.""" for x in range(1, len(tokens)): - self.duplicate().match(tokens[x], item) + self.duplicate(separate_names=True).match(tokens[x], item) with self.only_self(): self.match(tokens[0], item) diff --git a/coconut/root.py b/coconut/root.py index 60f3f9cd3..2e8a43034 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 13 +DEVELOP = 14 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index 8c45a5bb9..784f15eca 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -508,6 +508,8 @@ def main_test(): assert f"{{" == "{" assert f"}}" == "}" assert f"{1, 2}" == "(1, 2)" + match {"a": {"b": x }} or {"a": {"b": {"c": x}}} = {"a": {"b": {"c": "x"}}} + assert x == {"c": "x"} return True def tco_func() = tco_func() From 6d4116655bc612617fb3466d3304bdc9bc5e1848 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 1 Mar 2019 20:27:46 -0800 Subject: [PATCH 065/163] Perform minor cleanups --- coconut/compiler/compiler.py | 8 +++++--- coconut/compiler/grammar.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 17c477bdf..b4f8c11c5 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -370,9 +370,11 @@ def genhash(self, code, package_level=-1): """Generate a hash from code.""" return hex(checksum( hash_sep.join( - str(item) for item in (VERSION_STR,) - + self.__reduce__()[1] - + (package_level, code) + str(item) for item in ( + (VERSION_STR,) + + self.__reduce__()[1] + + (package_level, code) + ) ).encode(default_encoding), )) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 05c1a8bfe..cb568a7af 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -765,7 +765,6 @@ class Grammar(object): matrix_at = Forward() test = Forward() - namedexpr_test = Forward() test_no_chain, dubcolon = disable_inside(test, unsafe_dubcolon) test_no_infix, backtick = disable_inside(test, unsafe_backtick) @@ -874,6 +873,7 @@ class Grammar(object): dubstar_expr = Forward() comp_for = Forward() test_no_cond = Forward() + namedexpr_test = Forward() testlist = trace(itemlist(test, comma, suppress_trailing=False)) testlist_star_expr = trace(itemlist(test | star_expr, comma, suppress_trailing=False)) From b66e5f520485fa3850b4593a991e90f29dae7420 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 2 Mar 2019 16:47:44 -0800 Subject: [PATCH 066/163] Fix target docs --- DOCS.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/DOCS.md b/DOCS.md index ee67b6028..808d6119d 100644 --- a/DOCS.md +++ b/DOCS.md @@ -200,8 +200,9 @@ Finally, while Coconut will try to compile Python-3-specific syntax to its unive - keyword-only function arguments (use pattern-matching function definition instead), - destructuring assignment with `*`s (use pattern-matching instead), - tuples and lists with `*` unpacking or dicts with `**` unpacking (requires `--target 3.5`), -- `@` as matrix multiplication (requires `--target 3.5`), and -- `async` and `await` statements (requires `--target 3.5`). +- `@` as matrix multiplication (requires `--target 3.5`), +- `async` and `await` statements (requires `--target 3.5`), and +- `:=` assignment expressions (requires `--target 3.8`). ### Allowable Targets @@ -213,7 +214,8 @@ If the version of Python that the compiled code will be running on is known ahea - `3`, `3.2` (will work on any Python `>= 3.2`), - `3.3`, `3.4` (will work on any Python `>= 3.3`), - `3.5` (will work on any Python `>= 3.5`), -- `3.6` (will work on any Python `>= 3.6`), +- `3.6`, `3.7` (will work on any Python `>= 3.6`), +- `3.8` (will work on any Python `>= 3.8`), and - `sys` (chooses the target corresponding to the current Python version). _Note: Periods are ignored in target specifications, such that the target `27` is equivalent to the target `2.7`._ From 138a890312fdb09928a28e4800a31c1da694d0a7 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 2 Mar 2019 17:14:39 -0800 Subject: [PATCH 067/163] Fix assignment exprs Resolves #491. --- coconut/compiler/grammar.py | 21 ++++++++++----------- coconut/root.py | 2 +- tests/src/extras.coco | 1 + 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index cb568a7af..0af56807b 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -981,30 +981,29 @@ class Grammar(object): comma, )))) + call_item = ( + dubstar + test + | star + test + | name + default + | namedexpr_test + ) function_call_tokens = lparen.suppress() + ( # everything here must end with rparen rparen.suppress() | Group(op_item) + rparen.suppress() | Group(attach(addspace(test + comp_for), add_paren_handle)) + rparen.suppress() - | tokenlist(Group(dubstar + test | star + test | name + default | test), comma) + rparen.suppress() + | tokenlist(Group(call_item), comma) + rparen.suppress() ) function_call = attach(function_call_tokens, function_call_handle) questionmark_call_tokens = Group(tokenlist( Group( - questionmark | dubstar + test | star + test | name + default | test, + questionmark + | call_item, ), comma, )) methodcaller_args = ( - itemlist( - condense( - dubstar + test - | star + test - | name + default - | test, - ), - comma, - ) + itemlist(condense(call_item), comma) | op_item ) diff --git a/coconut/root.py b/coconut/root.py index 2e8a43034..3abbc78e7 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 14 +DEVELOP = 15 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/extras.coco b/tests/src/extras.coco index 5f31450f8..1c62a3f09 100644 --- a/tests/src/extras.coco +++ b/tests/src/extras.coco @@ -122,6 +122,7 @@ def test_extras(): assert parse("def f(*, x=None) = x") setup(target="3.8") assert parse("(a := b)") + assert parse("print(a := 1, b := 2)") if CoconutKernel is not None: if PY35: asyncio.set_event_loop(asyncio.new_event_loop()) From 0ad9b48945c4a174d3dd34fe6e72e17b81fccec0 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 3 Mar 2019 02:31:38 -0800 Subject: [PATCH 068/163] Add numpy special case to fmap Resolves #483. --- DOCS.md | 2 ++ coconut/compiler/templates/header.py_template | 3 +++ coconut/constants.py | 2 ++ coconut/root.py | 2 +- tests/src/cocotest/agnostic/main.coco | 4 ++++ 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/DOCS.md b/DOCS.md index 808d6119d..c2a667627 100644 --- a/DOCS.md +++ b/DOCS.md @@ -2103,6 +2103,8 @@ In functional programming, `fmap(func, obj)` takes a data type `obj` and returns For `dict`, or any other `collections.abc.Mapping`, `fmap` will be called on the mapping's `.items()` instead of the default iteration through its `.keys()`. +As an additional special case, for [`numpy`](http://www.numpy.org/) objects, `fmap` will use [`vectorize`](https://docs.scipy.org/doc/numpy/reference/generated/numpy.vectorize.html) to produce the result. + ##### Example **Coconut:** diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 4fe848857..20d2ae638 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -525,6 +525,9 @@ def makedata(data_type, *args): Override by defining obj.__fmap__(func).""" if _coconut.hasattr(obj, "__fmap__"): return obj.__fmap__(func) + if _coconut.hasattr(obj, "__class__") and _coconut.hasattr(obj.__class__, "__module__") and obj.__class__.__module__ == "numpy": + from numpy import vectorize + return vectorize(func)(obj) return _coconut_makedata(obj.__class__, *(_coconut_starmap(func, obj.items()) if _coconut.isinstance(obj, _coconut.abc.Mapping) else _coconut_map(func, obj))) def memoize(maxsize=None, *args, **kwargs): """Decorator that memoizes a function, diff --git a/coconut/constants.py b/coconut/constants.py index 40221cf6b..a805ff099 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -145,6 +145,7 @@ def checksum(data): "tests": ( "pytest", "pexpect", + "numpy", ), "cPyparsing": ( "cPyparsing", @@ -170,6 +171,7 @@ def checksum(data): "watchdog": (0, 9), "trollius": (2, 2), "requests": (2,), + "numpy": (1,), # don't upgrade this; it breaks on Python 2.6 "pytest": (3,), # don't upgrade this; it breaks on unix diff --git a/coconut/root.py b/coconut/root.py index 3abbc78e7..18829a33c 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 15 +DEVELOP = 16 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index 784f15eca..e59213a94 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -1,6 +1,9 @@ import sys import asyncio # type: ignore +import numpy as np + + def main_test(): """Basic no-dependency tests.""" assert "\n" == ( @@ -510,6 +513,7 @@ def main_test(): assert f"{1, 2}" == "(1, 2)" match {"a": {"b": x }} or {"a": {"b": {"c": x}}} = {"a": {"b": {"c": "x"}}} assert x == {"c": "x"} + assert np.all(fmap(-> _ + 1, np.arange(3)) == np.array([1, 2, 3])) return True def tco_func() = tco_func() From a268e23d0ff52d5f7cb096a626313222aee1fb6b Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 3 Mar 2019 02:34:55 -0800 Subject: [PATCH 069/163] Remove unnecessary checks --- coconut/compiler/templates/header.py_template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 20d2ae638..27c80d69e 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -525,7 +525,7 @@ def makedata(data_type, *args): Override by defining obj.__fmap__(func).""" if _coconut.hasattr(obj, "__fmap__"): return obj.__fmap__(func) - if _coconut.hasattr(obj, "__class__") and _coconut.hasattr(obj.__class__, "__module__") and obj.__class__.__module__ == "numpy": + if obj.__class__.__module__ == "numpy": from numpy import vectorize return vectorize(func)(obj) return _coconut_makedata(obj.__class__, *(_coconut_starmap(func, obj.items()) if _coconut.isinstance(obj, _coconut.abc.Mapping) else _coconut_map(func, obj))) From c08aee387224a4a1e4a9c1bf52cbcde56e5cfff0 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 3 Mar 2019 02:49:32 -0800 Subject: [PATCH 070/163] Increase line number limit. Resolves #391, #462. --- coconut/compiler/compiler.py | 7 ++++--- coconut/root.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index b4f8c11c5..6551a403c 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -945,12 +945,13 @@ def reind_proc(self, inputstring, **kwargs): return "\n".join(out) def ln_comment(self, ln): - """Get an end line comment. CoconutInternalExceptions should always be caught and complained.""" + """Get an end line comment.""" + # CoconutInternalExceptions should always be caught and complained here if self.keep_lines: - if not 1 <= ln <= len(self.original_lines) + 1: + if not 1 <= ln <= len(self.original_lines) + 2: complain(CoconutInternalException( "out of bounds line number", ln, - "not in range [1, " + str(len(self.original_lines) + 1) + "]", + "not in range [1, " + str(len(self.original_lines) + 2) + "]", )) if ln >= len(self.original_lines) + 1: # trim too large lni = -1 diff --git a/coconut/root.py b/coconut/root.py index 18829a33c..b742cb529 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 16 +DEVELOP = 17 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From d5491d28a1391ae958014d0f5631555f3f2e4eaa Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 3 Mar 2019 16:01:26 -0800 Subject: [PATCH 071/163] Add kwargs pipes Resolves #323. --- DOCS.md | 18 ++- coconut/compiler/compiler.py | 8 ++ coconut/compiler/grammar.py | 110 ++++++++++++------ coconut/compiler/header.py | 2 +- coconut/compiler/templates/header.py_template | 27 +++-- coconut/root.py | 4 +- tests/src/cocotest/agnostic/main.coco | 3 +- tests/src/cocotest/agnostic/suite.coco | 23 ++++ 8 files changed, 142 insertions(+), 53 deletions(-) diff --git a/DOCS.md b/DOCS.md index c2a667627..aaaf40f66 100644 --- a/DOCS.md +++ b/DOCS.md @@ -446,8 +446,10 @@ Coconut uses pipe operators for pipeline-style function application. All the ope ```coconut (|>) => pipe forward (|*>) => multiple-argument pipe forward +(|**>) => keyword argument pipe forward (<|) => pipe backward (<*|) => multiple-argument pipe backward +(<**|) => keyword argument pipe backward ``` Additionally, all pipe operators support a lambda as the last argument, despite lambdas having a lower precedence. Thus, `a |> x -> b |> c` is equivalent to `a |> (x -> b |> c)`, not `a |> (x -> b) |> c`. @@ -487,11 +489,11 @@ print(sq(operator.add(1, 2))) Coconut has three basic function composition operators: `..`, `..>`, and `<..`. Both `..` and `<..` use math-style "backwards" function composition, where the first function is called last, while `..>` uses "forwards" function composition, where the first function is called first. -The `..>` and `<..` function composition pipe operators also have `..*>` and `<*..` forms which are, respectively, the equivalents of `|*>` and `<*|`. Forwards and backwards function composition pipes cannot be used together in the same expression (unlike normal pipes) and have precedence in-between `None`-coalescing and normal pipes. +The `..>` and `<..` function composition pipe operators also have `..*>` and `<*..` forms which are, respectively, the equivalents of `|*>` and `<*|` and `..**>` and `<**..` forms correspond to `|**>` and `<**|`. Forwards and backwards function composition pipes cannot be used together in the same expression (unlike normal pipes) and have precedence in-between `None`-coalescing and normal pipes. The `..` operator has lower precedence than attribute access, slices, function calls, etc., but higher precedence than all other operations. -The in-place function composition operators are `..=`, `..>=`, `<..=`, `..*>=`, and `<*..=`. +The in-place function composition operators are `..=`, `..>=`, `<..=`, `..*>=`, `<*..=`, `..**>`, and `..**>`. ##### Example @@ -667,9 +669,11 @@ Coconut supports Unicode alternatives to many different operator symbols. The Un ``` → (\u2192) => "->" ↦ (\u21a6) => "|>" -*↦ (*\u21a6) => "|*>" ↤ (\u21a4) => "<|" +*↦ (*\u21a6) => "|*>" ↤* (\u21a4*) => "<*|" +**↦ (**\u21a6) => "|**>" +↤** (\u21a4**) => "<**|" × (\xd7) => "*" ↑ (\u2191) => "**" ÷ (\xf7) => "/" @@ -679,6 +683,8 @@ Coconut supports Unicode alternatives to many different operator symbols. The Un <∘ (<\u2218) => "<.." ∘*> (\u2218*>) => "..*>" <*∘ (<*\u2218) => "<*.." +∘**> (\u2218**>) => "..**>" +<**∘ (<**\u2218) => "<**.." − (\u2212) => "-" (only subtraction) ⁻ (\u207b) => "-" (only negation) ¬ (\xac) => "~" @@ -1133,13 +1139,17 @@ A very common thing to do in functional programming is to make use of function v ```coconut (|>) => # pipe forward -(|*>) => # multi-arg pipe forward (<|) => # pipe backward +(|*>) => # multi-arg pipe forward (<*|) => # multi-arg pipe backward +(|**>) => # keyword arg pipe forward +(<**|) => # keyword arg pipe backward (..), (<..) => # backward function composition (..>) => # forward function composition (<*..) => # multi-arg backward function composition (..*>) => # multi-arg forward function composition +(<**..) => # keyword arg backward function composition +(..**>) => # keyword arg forward function composition (.) => (getattr) (::) => (itertools.chain) # will not evaluate its arguments lazily ($) => (functools.partial) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 6551a403c..223048219 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -1156,10 +1156,14 @@ def augassign_handle(self, tokens): out += name + " = (" + item + ")(" + name + ")" elif op == "|*>=": out += name + " = (" + item + ")(*" + name + ")" + elif op == "|**>=": + out += name + " = (" + item + ")(**" + name + ")" elif op == "<|=": out += name + " = " + name + "((" + item + "))" elif op == "<*|=": out += name + " = " + name + "(*(" + item + "))" + elif op == "<**|=": + out += name + " = " + name + "(**(" + item + "))" elif op == "..=" or op == "<..=": out += name + " = _coconut_forward_compose((" + item + "), " + name + ")" elif op == "..>=": @@ -1168,6 +1172,10 @@ def augassign_handle(self, tokens): out += name + " = _coconut_forward_star_compose((" + item + "), " + name + ")" elif op == "..*>=": out += name + " = _coconut_forward_star_compose(" + name + ", (" + item + "))" + elif op == "<**..=": + out += name + " = _coconut_forward_dubstar_compose((" + item + "), " + name + ")" + elif op == "..**>=": + out += name + " = _coconut_forward_dubstar_compose(" + name + ", (" + item + "))" elif op == "??=": out += name + " = " + item + " if " + name + " is None else " + name elif op == "::=": diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 0af56807b..99acb99a8 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -183,18 +183,23 @@ def get_infix_items(tokens, callback=infix_error): return func, args -def comp_pipe_info(op): - """Returns (direction, star) where direction is 'forwards' or 'backwards'.""" - if op == "..>": - return "forwards", False - elif op == "<..": - return "backwards", False - elif op == "..*>": - return "forwards", True - elif op == "<*..": - return "backwards", True +def pipe_info(op): + """Returns (direction, stars) where direction is 'forwards' or 'backwards'. + Works with normal pipe operators and composition pipe operators.""" + if op.startswith("<**"): + return "backwards", 2 + elif op.endswith("**>"): + return "forwards", 2 + elif op.endswith("*>"): + return "forwards", 1 + elif op.startswith("<*"): + return "backwards", 1 + elif op.endswith(">"): + return "forwards", 0 + elif op.startswith("<"): + return "backwards", 0 else: - raise CoconutInternalException("invalid function composition pipe operator", op) + raise CoconutInternalException("invalid pipe operator", op) # end: HELPERS @@ -314,25 +319,26 @@ def pipe_handle(loc, tokens, **kwargs): else: item, op = tokens.pop(), tokens.pop() + direction, stars = pipe_info(op) + star_str = "*" * stars - if op == "|>" or op == "|*>": + if direction == "forwards": # if this is an implicit partial, we have something to apply it to, so optimize it name, split_item = pipe_item_split(item, loc) - star = "*" if op == "|*>" else "" if name == "expr": internal_assert(len(split_item) == 1) - return "(" + split_item[0] + ")(" + star + pipe_handle(loc, tokens) + ")" + return "(" + split_item[0] + ")(" + star_str + pipe_handle(loc, tokens) + ")" elif name == "partial": internal_assert(len(split_item) == 3) - return split_item[0] + "(" + join_args((split_item[1], star + pipe_handle(loc, tokens), split_item[2])) + ")" + return split_item[0] + "(" + join_args((split_item[1], star_str + pipe_handle(loc, tokens), split_item[2])) + ")" elif name == "attrgetter": internal_assert(len(split_item) == 2) - if star: + if stars: raise CoconutDeferredSyntaxError("cannot star pipe into attribute access or method call", loc) return "(" + pipe_handle(loc, tokens) + ")." + split_item[0] + ("(" + split_item[1] + ")" if split_item[1] is not None else "") elif name == "itemgetter": internal_assert(len(split_item) == 2) - if star: + if stars: raise CoconutDeferredSyntaxError("cannot star pipe into item getting", loc) op, args = split_item if op == "[": @@ -344,38 +350,37 @@ def pipe_handle(loc, tokens, **kwargs): else: raise CoconutInternalException("invalid split pipe item", split_item) - elif op == "<|" or op == "<*|": + elif direction == "backwards": # for backwards pipes, we just reuse the machinery for forwards pipes - star = "*" if op == "<*|" else "" inner_item = pipe_handle(loc, tokens, top=False) if isinstance(inner_item, str): inner_item = [inner_item] # artificial pipe item - return pipe_handle(loc, [item, "|" + star + ">", inner_item]) + return pipe_handle(loc, [item, "|" + star_str + ">", inner_item]) else: - raise CoconutInternalException("invalid pipe operator", op) + raise CoconutInternalException("invalid pipe operator direction", direction) def comp_pipe_handle(loc, tokens): """Process pipe function composition.""" internal_assert(len(tokens) >= 3 and len(tokens) % 2 == 1, "invalid composition pipe tokens", tokens) funcs = [tokens[0]] - stars = [] + stars_per_func = [] direction = None for i in range(1, len(tokens), 2): op, fn = tokens[i], tokens[i + 1] - new_direction, star = comp_pipe_info(op) + new_direction, stars = pipe_info(op) if direction is None: direction = new_direction elif new_direction != direction: raise CoconutDeferredSyntaxError("cannot mix function composition pipe operators with different directions", loc) funcs.append(fn) - stars.append(star) + stars_per_func.append(stars) if direction == "backwards": funcs.reverse() - stars.reverse() + stars_per_func.reverse() func = funcs.pop(0) - funcstars = zip(funcs, stars) + funcstars = zip(funcs, stars_per_func) return "_coconut_base_compose(" + func + ", " + ", ".join( "(%s, %s)" % (f, star) for f, star in funcstars ) + ")" @@ -707,7 +712,7 @@ class Grammar(object): rbrack = Literal("]") lbrace = Literal("{") rbrace = Literal("}") - lbanana = ~Literal("(|)") + ~Literal("(|>)") + ~Literal("(|*>)") + Literal("(|") + lbanana = Literal("(|") + ~Word(")>*", exact=1) rbanana = Literal("|)") lparen = ~lbanana + Literal("(") rparen = Literal(")") @@ -718,9 +723,11 @@ class Grammar(object): dubslash = Literal("//") slash = ~dubslash + Literal("/") pipe = Literal("|>") | fixto(Literal("\u21a6"), "|>") - star_pipe = Literal("|*>") | fixto(Literal("*\u21a6"), "|*>") back_pipe = Literal("<|") | fixto(Literal("\u21a4"), "<|") - back_star_pipe = Literal("<*|") | fixto(Literal("\u21a4*"), "<*|") + star_pipe = Literal("|*>") | fixto(Literal("*\u21a6"), "|*>") + back_star_pipe = Literal("<*|") | ~Literal("\u21a4**") + fixto(Literal("\u21a4*"), "<*|") + dubstar_pipe = Literal("|**>") | fixto(Literal("**\u21a6"), "|**>") + back_dubstar_pipe = Literal("<**|") | fixto(Literal("\u21a4**"), "<**|") dotdot = ( ~Literal("...") + ~Literal("..>") + ~Literal("..*>") + Literal("..") | ~Literal("\u2218>") + ~Literal("\u2218*>") + fixto(Literal("\u2218"), "..") @@ -729,6 +736,8 @@ class Grammar(object): comp_back_pipe = Literal("<..") | fixto(Literal("<\u2218"), "<..") comp_star_pipe = Literal("..*>") | fixto(Literal("\u2218*>"), "..*>") comp_back_star_pipe = Literal("<*..") | fixto(Literal("<*\u2218"), "<*..") + comp_dubstar_pipe = Literal("..**>") | fixto(Literal("\u2218**>"), "..**>") + comp_back_dubstar_pipe = Literal("<**..") | fixto(Literal("<**\u2218"), "<**..") amp = Literal("&") | fixto(Literal("\u2227") | Literal("\u2229"), "&") caret = Literal("^") | fixto(Literal("\u22bb") | Literal("\u2295"), "^") unsafe_bar = ~Literal("|>") + ~Literal("|*>") + Literal("|") | fixto(Literal("\u2228") | Literal("\u222a"), "|") @@ -835,14 +844,18 @@ class Grammar(object): augassign = ( Combine(pipe + equals) - | Combine(star_pipe + equals) | Combine(back_pipe + equals) + | Combine(star_pipe + equals) | Combine(back_star_pipe + equals) - | Combine(dotdot + equals) + | Combine(dubstar_pipe + equals) + | Combine(back_dubstar_pipe + equals) | Combine(comp_pipe + equals) + | Combine(dotdot + equals) | Combine(comp_back_pipe + equals) | Combine(comp_star_pipe + equals) | Combine(comp_back_star_pipe + equals) + | Combine(comp_dubstar_pipe + equals) + | Combine(comp_back_dubstar_pipe + equals) | Combine(unsafe_dubcolon + equals) | Combine(div_dubslash + equals) | Combine(div_slash + equals) @@ -897,14 +910,23 @@ class Grammar(object): test_expr = yield_expr | testlist_star_expr op_item = ( - fixto(pipe, "_coconut_pipe") + + # must go dubstar then star then no star + fixto(dubstar_pipe, "_coconut_dubstar_pipe") + | fixto(back_dubstar_pipe, "_coconut_back_dubstar_pipe") | fixto(star_pipe, "_coconut_star_pipe") - | fixto(back_pipe, "_coconut_back_pipe") | fixto(back_star_pipe, "_coconut_back_star_pipe") - | fixto(dotdot | comp_back_pipe, "_coconut_back_compose") - | fixto(comp_pipe, "_coconut_forward_compose") + | fixto(pipe, "_coconut_pipe") + | fixto(back_pipe, "_coconut_back_pipe") + + # must go dubstar then star then no star + | fixto(comp_dubstar_pipe, "_coconut_forward_dubstar_compose") + | fixto(comp_back_dubstar_pipe, "_coconut_back_dubstar_compose") | fixto(comp_star_pipe, "_coconut_forward_star_compose") | fixto(comp_back_star_pipe, "_coconut_back_star_compose") + | fixto(comp_pipe, "_coconut_forward_compose") + | fixto(dotdot | comp_back_pipe, "_coconut_back_compose") + | fixto(keyword("and"), "_coconut_bool_and") | fixto(keyword("or"), "_coconut_bool_or") | fixto(dubquestion, "_coconut_none_coalesce") @@ -1176,7 +1198,14 @@ class Grammar(object): none_coalesce_expr = attach(infix_expr + ZeroOrMore(dubquestion.suppress() + infix_expr), none_coalesce_handle) - comp_pipe_op = comp_pipe | comp_back_pipe | comp_star_pipe | comp_back_star_pipe + comp_pipe_op = ( + comp_pipe + | comp_star_pipe + | comp_back_pipe + | comp_back_star_pipe + | comp_dubstar_pipe + | comp_back_dubstar_pipe + ) comp_pipe_expr = ( none_coalesce_expr + ~comp_pipe_op | attach( @@ -1185,7 +1214,14 @@ class Grammar(object): ) ) - pipe_op = pipe | star_pipe | back_pipe | back_star_pipe + pipe_op = ( + pipe + | back_pipe + | star_pipe + | back_star_pipe + | dubstar_pipe + | back_dubstar_pipe + ) pipe_item = ( # we need the pipe_op since any of the atoms could otherwise be the start of an expression Group(attrgetter_atom_tokens("attrgetter")) + pipe_op diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index ccd030f35..ceacf414a 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -202,7 +202,7 @@ def NamedTuple(name, fields): return _coconut.collections.namedtuple(name, [x for x, t in fields])'''.format(**format_dict), ) - format_dict["underscore_imports"] = "_coconut, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_pipe, _coconut_star_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_addpattern, _coconut_sentinel".format(**format_dict) + format_dict["underscore_imports"] = "_coconut, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_back_pipe, _coconut_star_pipe, _coconut_back_star_pipe, _coconut_dubstar_pipe, _coconut_back_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_addpattern, _coconut_sentinel".format(**format_dict) # ._coconut_tco_func is used in main.coco, so don't remove it # here without replacing its usage there diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 27c80d69e..619f5112c 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -29,29 +29,40 @@ class _coconut_base_compose{object}: def __init__(self, func, *funcstars): self.func = func self.funcstars = [] - for f, star in funcstars: + for f, stars in funcstars: if isinstance(f, _coconut_base_compose): - self.funcstars.append((f.func, star)) + self.funcstars.append((f.func, stars)) self.funcstars += f.funcstars else: - self.funcstars.append((f, star)) + self.funcstars.append((f, stars)) def __call__(self, *args, **kwargs): arg = self.func(*args, **kwargs) - for f, star in self.funcstars: - arg = f(*arg) if star else f(arg) + for f, stars in self.funcstars: + if stars == 0: + arg = f(arg) + elif stars == 1: + arg = f(*arg) + elif stars == 2: + arg = f(**arg) + else: + raise ValueError() return arg def __repr__(self): - return _coconut.repr(self.func) + " " + " ".join(("..*> " if star else "..> ") + _coconut.repr(f) for f, star in self.funcstars) + return _coconut.repr(self.func) + " " + " ".join(("..*> " if star == 1 else "..**>" if star == 2 else "..> ") + _coconut.repr(f) for f, star in self.funcstars) def __reduce__(self): return (self.__class__, (self.func,) + _coconut.tuple(self.funcstars)) -def _coconut_forward_compose(func, *funcs): return _coconut_base_compose(func, *((f, False) for f in funcs)) +def _coconut_forward_compose(func, *funcs): return _coconut_base_compose(func, *((f, 0) for f in funcs)) def _coconut_back_compose(*funcs): return _coconut_forward_compose(*_coconut.reversed(funcs)) -def _coconut_forward_star_compose(func, *funcs): return _coconut_base_compose(func, *((f, True) for f in funcs)) +def _coconut_forward_star_compose(func, *funcs): return _coconut_base_compose(func, *((f, 1) for f in funcs)) def _coconut_back_star_compose(*funcs): return _coconut_forward_star_compose(*_coconut.reversed(funcs)) +def _coconut_forward_dubstar_compose(func, *funcs): return _coconut_base_compose(func, *((f, 2) for f in funcs)) +def _coconut_back_dubstar_compose(*funcs): return _coconut_forward_dubstar_compose(*_coconut.reversed(funcs)) def _coconut_pipe(x, f): return f(x) def _coconut_star_pipe(xs, f): return f(*xs) +def _coconut_dubstar_pipe(kws, f): return f(**kws) def _coconut_back_pipe(f, x): return f(x) def _coconut_back_star_pipe(f, xs): return f(*xs) +def _coconut_back_dubstar_pipe(f, kws): return f(**kws) def _coconut_bool_and(a, b): return a and b def _coconut_bool_or(a, b): return a or b def _coconut_none_coalesce(a, b): return a if a is not None else b diff --git a/coconut/root.py b/coconut/root.py index b742cb529..c5153ea26 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -40,12 +40,12 @@ PY26 = _coconut_sys.version_info < (2, 7) PY3_HEADER = r'''from builtins import chr, filter, hex, input, int, map, object, oct, open, print, range, str, zip, filter, reversed, enumerate -py_chr, py_hex, py_input, py_int, py_map, py_object, py_oct, py_open, py_print, py_range, py_str, py_zip, py_filter, py_reversed, py_enumerate = chr, hex, input, int, map, object, oct, open, print, range, str, zip, filter, reversed, enumerate +py_chr, py_hex, py_input, py_int, py_map, py_object, py_oct, py_open, py_print, py_range, py_str, py_zip, py_filter, py_reversed, py_enumerate, py_repr = chr, hex, input, int, map, object, oct, open, print, range, str, zip, filter, reversed, enumerate, repr _coconut_str = str ''' PY27_HEADER = r'''from __builtin__ import chr, filter, hex, input, int, map, object, oct, open, print, range, str, zip, filter, reversed, enumerate, raw_input, xrange -py_chr, py_hex, py_input, py_int, py_map, py_object, py_oct, py_open, py_print, py_range, py_str, py_zip, py_filter, py_reversed, py_enumerate, py_raw_input, py_xrange = chr, hex, input, int, map, object, oct, open, print, range, str, zip, filter, reversed, enumerate, raw_input, xrange +py_chr, py_hex, py_input, py_int, py_map, py_object, py_oct, py_open, py_print, py_range, py_str, py_zip, py_filter, py_reversed, py_enumerate, py_raw_input, py_xrange, py_repr = chr, hex, input, int, map, object, oct, open, print, range, str, zip, filter, reversed, enumerate, raw_input, xrange, repr _coconut_NotImplemented, _coconut_raw_input, _coconut_xrange, _coconut_int, _coconut_long, _coconut_print, _coconut_str, _coconut_unicode, _coconut_repr = NotImplemented, raw_input, xrange, int, long, print, str, unicode, repr from future_builtins import * chr, str = unichr, unicode diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index e59213a94..d6d3eedc0 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -506,7 +506,7 @@ def main_test(): x = 1 y = "2" assert f"{x} == {y}" == "1 == 2" - assert f"{x!r} == {y!r}" == "1 == '2'" + assert f"{x!r} == {y!r}" == "1 == " + py_repr("2") assert f"{({})}" == "{}" == f"{({})!r}" assert f"{{" == "{" assert f"}}" == "}" @@ -514,6 +514,7 @@ def main_test(): match {"a": {"b": x }} or {"a": {"b": {"c": x}}} = {"a": {"b": {"c": "x"}}} assert x == {"c": "x"} assert np.all(fmap(-> _ + 1, np.arange(3)) == np.array([1, 2, 3])) + assert py_repr("x") == ("u'x'" if sys.version_info < (3,) else "'x'") return True def tco_func() = tco_func() diff --git a/tests/src/cocotest/agnostic/suite.coco b/tests/src/cocotest/agnostic/suite.coco index 8ef9deca5..d0fc6f9df 100644 --- a/tests/src/cocotest/agnostic/suite.coco +++ b/tests/src/cocotest/agnostic/suite.coco @@ -512,6 +512,29 @@ def suite_test(): assert err else: assert False + + assert dict(a=1, b=2) |**> ret_args_kwargs == ((), dict(a=1, b=2)) == (|**>)(dict(a=1, b=2), ret_args_kwargs) + x = dict(a=1, b=2) + x |**>= ret_args_kwargs + assert x == ((), dict(a=1, b=2)) + + assert ret_args_kwargs <**| {"a": 1, "b": 2} == ((), {"a": 1, "b": 2}) == (<**|)(ret_args_kwargs, {"a": 1, "b": 2}) + f = ret_args_kwargs + f <**|= {"a": 1, "b": 2} + assert f == ((), {"a": 1, "b": 2}) + + ret_dict = -> dict(x=2) + + assert (ret_dict ..**> ret_args_kwargs$(1))() == ((1,), dict(x=2)) == ((..**>)(ret_dict, ret_args_kwargs$(1)))() + x = ret_dict + x ..**>= ret_args_kwargs$(1) + assert x() == ((1,), dict(x=2)) + + assert (ret_args_kwargs$(1) <**.. ret_dict)() == ((1,), dict(x=2)) == ((<**..)(ret_args_kwargs$(1), ret_dict))() + f = ret_args_kwargs$(1) + f <**..= ret_dict + assert f() == ((1,), dict(x=2)) + return True def tco_test(): From c434e063d441a424684a571237cca8c782c2fef1 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 3 Mar 2019 17:23:46 -0800 Subject: [PATCH 072/163] Clean up docs --- DOCS.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/DOCS.md b/DOCS.md index aaaf40f66..72496339d 100644 --- a/DOCS.md +++ b/DOCS.md @@ -316,7 +316,7 @@ In order of precedence, highest first, the operators supported in Coconut are: ===================== ========================== Symbol(s) Associativity ===================== ========================== -.. n/a won't capture call +.. n/a (won't capture call) ** right +, -, ~ unary *, /, //, %, @ left @@ -325,24 +325,26 @@ Symbol(s) Associativity & left ^ left | left -:: n/a lazy -a `b` c left captures lambda -?? left short-circuit -..>, <.., ..*>, <*.. n/a captures lambda -|>, <|, |*>, <*| left captures lambda +:: n/a (lazy) +a `b` c left (captures lambda) +?? left (short-circuits) +..>, <.., ..*>, <*.., n/a (captures lambda) + ..**>, <**.. +|>, <|, |*>, <*|, left (captures lambda) + |**>, <**| ==, !=, <, >, <=, >=, in, not in, is, is not n/a not unary -and left short-circuit -or left short-circuit -a if b else c ternary left short-circuit +and left (short-circuits) +or left (short-circuits) +a if b else c ternary left (short-circuits) -> right ===================== ========================== ``` -Note that because indexing has a greater precedence than piping, expressions of the form `x |> y[0]` are equivalent to `x |> (y[0])`. +Note that because addition has a greater precedence than piping, expressions of the form `x |> y + z` are equivalent to `x |> (y + z)`. ### Lambdas @@ -468,7 +470,7 @@ func(args, obj.attribute.method(args))[index] ``` where `func` has to go at the beginning. -If Coconut compiled each of the partials in the pipe syntax as an actual partial application object, it would make the Coconut-style syntax here so much slower than the Python-style syntax as to make it near-useless. Thus, Coconut does not do that. If any of the above styles of partials or implicit partials are used in pipes, they will whenever possible be compiled to the Python-style syntax, producing no intermediate partial application objects. +If Coconut compiled each of the partials in the pipe syntax as an actual partial application object, it would make the Coconut-style syntax significantly slower than the Python-style syntax. Thus, Coconut does not do that. If any of the above styles of partials or implicit partials are used in pipes, they will whenever possible be compiled to the Python-style syntax, producing no intermediate partial application objects. ##### Example @@ -487,11 +489,9 @@ print(sq(operator.add(1, 2))) ### Compose -Coconut has three basic function composition operators: `..`, `..>`, and `<..`. Both `..` and `<..` use math-style "backwards" function composition, where the first function is called last, while `..>` uses "forwards" function composition, where the first function is called first. +Coconut has three basic function composition operators: `..`, `..>`, and `<..`. Both `..` and `<..` use math-style "backwards" function composition, where the first function is called last, while `..>` uses "forwards" function composition, where the first function is called first. Forwards and backwards function composition pipes cannot be used together in the same expression (unlike normal pipes) and have precedence in-between `None`-coalescing and normal pipes. The `..>` and `<..` function composition pipe operators also have `..*>` and `<*..` forms which are, respectively, the equivalents of `|*>` and `<*|` and `..**>` and `<**..` forms which correspond to `|**>` and `<**|`. -The `..>` and `<..` function composition pipe operators also have `..*>` and `<*..` forms which are, respectively, the equivalents of `|*>` and `<*|` and `..**>` and `<**..` forms correspond to `|**>` and `<**|`. Forwards and backwards function composition pipes cannot be used together in the same expression (unlike normal pipes) and have precedence in-between `None`-coalescing and normal pipes. - -The `..` operator has lower precedence than attribute access, slices, function calls, etc., but higher precedence than all other operations. +The `..` operator has lower precedence than attribute access, slices, function calls, etc., but higher precedence than all other operations while the `..>` pipe operators have a precedence directly higher than normal pipes. The in-place function composition operators are `..=`, `..>=`, `<..=`, `..*>=`, `<*..=`, `..**>`, and `..**>`. @@ -563,8 +563,7 @@ _Can't be done without a complicated iterator slicing function and inspection of Coconut provides `??` as a `None`-coalescing operator, similar to the `??` null-coalescing operator in C# and Swift. Additionally, Coconut implements all of the `None`-aware operators proposed in [PEP 505](https://www.python.org/dev/peps/pep-0505/). -Coconut's `??` operator evaluates to its left operand if that operand is not `None`, otherwise its right operand. The expression `foo ?? bar` evaluates to `foo` as long as it isn't `None`, and to `bar` if it is. -The `None`-coalescing operator is short-circuiting, such that if the left operand is not `None`, the right operand won't be evaluated. This allows the right operand to be a potentially expensive operation without incurring any unnecessary cost. +Coconut's `??` operator evaluates to its left operand if that operand is not `None`, otherwise its right operand. The expression `foo ?? bar` evaluates to `foo` as long as it isn't `None`, and to `bar` if it is. The `None`-coalescing operator is short-circuiting, such that if the left operand is not `None`, the right operand won't be evaluated. This allows the right operand to be a potentially expensive operation without incurring any unnecessary cost. The `None`-coalescing operator has a precedence in-between infix function calls and composition pipes, and is left-associative. @@ -579,6 +578,7 @@ could_be_none() ?? calculate_default_value() ```coconut_python (lambda result: result if result is not None else calculate_default_value())(could_be_none()) ``` + #### Coalescing Assignment Operator The in-place assignment operator is `??=`, which allows conditionally setting a variable if it is currently `None`. @@ -597,7 +597,7 @@ baz = 0 baz ??= expensive_task() # right hand side isn't evaluated ``` -#### Other None-aware Operators +#### Other None-Aware Operators Coconut also allows a single `?` before attribute access, function calling, partial application, and (iterator) indexing to short-circuit the rest of the evaluation if everything so far evaluates to `None`. This is sometimes known as a "safe navigation" operator. @@ -630,7 +630,7 @@ import functools ### Expanded Indexing for Iterables -Beyond indexing standard Python sequences, Coconut supports indexing into a number of iterables, including `range` and `map`, which do not support random access in Python. In Coconut, indexing into an iterable of this type uses the same syntax as indexing into a sequence in vanilla Python. +Beyond indexing standard Python sequences, Coconut supports indexing into a number of iterables, including `range` and `map`, which do not support random access in all Python versions but do in Coconut. In Coconut, indexing into an iterable of this type uses the same syntax as indexing into a sequence in vanilla Python. ##### Example @@ -880,7 +880,7 @@ pattern ::= ( - Iterator Splits (` :: `): will match the beginning of an iterable (`collections.abc.Iterable`) against the ``, then bind the rest to `` or check that the iterable is done. - Complex String Matching (` + + `): matches strings that start and end with the given substrings, binding the middle to ``. -_Note: Like [iterator slicing](#iterator-slicing), iterator and lazy list matching make no guarantee that the original iterator matched against be preserved (to preserve the iterator, use Coconut's [`tee`](#tee) or [`reiterable`](#reiterable) built-in)._ +_Note: Like [iterator slicing](#iterator-slicing), iterator and lazy list matching make no guarantee that the original iterator matched against be preserved (to preserve the iterator, use Coconut's [`tee`](#tee) or [`reiterable`](#reiterable) built-ins)._ When checking whether or not an object can be matched against in a particular fashion, Coconut makes use of Python's abstract base classes. Therefore, to enable proper matching for a custom object, register it with the proper abstract base classes. From 511fd39048b95f9750b6916320e4461c80d133a4 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 5 Mar 2019 02:24:21 -0800 Subject: [PATCH 073/163] Add annotations future importing --- DOCS.md | 6 ++++-- coconut/compiler/header.py | 2 ++ coconut/constants.py | 14 +++++++++++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/DOCS.md b/DOCS.md index 72496339d..8eb1d7a9c 100644 --- a/DOCS.md +++ b/DOCS.md @@ -212,9 +212,11 @@ If the version of Python that the compiled code will be running on is known ahea - `2`, `2.6` (will work on any Python `>= 2.6` but `< 3`), - `2.7` (will work on any Python `>= 2.7` but `< 3`), - `3`, `3.2` (will work on any Python `>= 3.2`), -- `3.3`, `3.4` (will work on any Python `>= 3.3`), +- `3.3` (will work on any Python `>= 3.3`), +- `3.4` (will work on any Python `>= 3.4`), - `3.5` (will work on any Python `>= 3.5`), -- `3.6`, `3.7` (will work on any Python `>= 3.6`), +- `3.6` (will work on any Python `>= 3.6`), +- `3.7` (will work on any Python `>= 3.7`), - `3.8` (will work on any Python `>= 3.8`), and - `sys` (chooses the target corresponding to the current Python version). diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index ceacf414a..3a7f71ef2 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -274,6 +274,8 @@ def getheader(which, target="", use_hash=None, no_tco=False, strict=False): if target_startswith != "3": header += "from __future__ import print_function, absolute_import, unicode_literals, division\n" + elif target_info >= (3, 7): + header += "from __future__ import generator_stop, annotations\n" elif target_info >= (3, 5): header += "from __future__ import generator_stop\n" diff --git a/coconut/constants.py b/coconut/constants.py index a805ff099..3114f650d 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -338,13 +338,21 @@ def checksum(data): py2_vers = ((2, 6), (2, 7)) py3_vers = ((3, 2), (3, 3), (3, 4), (3, 5), (3, 6), (3, 7)) -specific_targets = ("2", "27", "3", "33", "35", "36", "38") +specific_targets = ( + "2", + "27", + "3", + "33", + "34", + "35", + "36", + "37", + "38", +) pseudo_targets = { "universal": "", "26": "2", "32": "3", - "34": "33", - "37": "36", } targets = ("",) + specific_targets From aebfeb7d0e916363bab4f65040ea199aca3e4599 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 5 Mar 2019 13:48:45 -0800 Subject: [PATCH 074/163] Truncate pattern errors Resolves #493. --- coconut/compiler/compiler.py | 11 ++++++++--- coconut/constants.py | 3 +++ coconut/root.py | 2 +- tests/src/cocotest/agnostic/main.coco | 5 +++++ 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 223048219..6624a6d7f 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -74,6 +74,8 @@ function_match_error_var, legal_indent_chars, format_var, + match_val_repr_var, + max_match_val_repr_len, ) from coconut.exceptions import ( CoconutException, @@ -1431,11 +1433,14 @@ def pattern_error(self, original, loc, value_var, check_var, match_error_class=' repr_wrap = self.wrap_str_of(ascii(base_line)) return ( "if not " + check_var + ":\n" + openindent + + match_val_repr_var + " = _coconut.repr(" + value_var + ")\n" + match_err_var + " = " + match_error_class + '("pattern-matching failed for " ' - + repr_wrap + ' " in " + _coconut.repr(_coconut.repr(' + value_var + ")))\n" + + repr_wrap + ' " in " + (' + match_val_repr_var + + " if _coconut.len(" + match_val_repr_var + ") <= " + str(max_match_val_repr_len) + + " else " + match_val_repr_var + "[:" + str(max_match_val_repr_len) + '] + "..."))\n' + match_err_var + ".pattern = " + line_wrap + "\n" - + match_err_var + ".value = " + value_var - + "\nraise " + match_err_var + "\n" + closeindent + + match_err_var + ".value = " + value_var + "\n" + + "raise " + match_err_var + "\n" + closeindent ) def destructuring_stmt_handle(self, original, loc, tokens): diff --git a/coconut/constants.py b/coconut/constants.py index 3114f650d..f0ddbb19b 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -379,6 +379,8 @@ def checksum(data): justify_len = 79 # ideal line length +max_match_val_repr_len = 500 # max len of match val reprs in err msgs + reserved_prefix = "_coconut" decorator_var = reserved_prefix + "_decorator" lazy_chain_var = reserved_prefix + "_lazy_chain" @@ -401,6 +403,7 @@ def checksum(data): match_check_var = reserved_prefix + "_match_check" match_temp_var = reserved_prefix + "_match_temp" match_err_var = reserved_prefix + "_match_err" +match_val_repr_var = reserved_prefix + "_match_val_repr" case_check_var = reserved_prefix + "_case_check" function_match_error_var = reserved_prefix + "_FunctionMatchError" diff --git a/coconut/root.py b/coconut/root.py index c5153ea26..512b21e92 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 17 +DEVELOP = 18 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index d6d3eedc0..ea3446622 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -515,6 +515,11 @@ def main_test(): assert x == {"c": "x"} assert np.all(fmap(-> _ + 1, np.arange(3)) == np.array([1, 2, 3])) assert py_repr("x") == ("u'x'" if sys.version_info < (3,) else "'x'") + def foo(x is int) = x + try: + foo(["foo"] * 100000) + except MatchError as err: + assert len(repr(err)) < 1000 return True def tco_func() = tco_func() From f9aaf7cad2a7bca41a79add528b3b10f5690a1f4 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 6 Mar 2019 00:01:45 -0800 Subject: [PATCH 075/163] Fix tests --- .travis.yml | 1 - DOCS.md | 4 +--- tests/src/cocotest/agnostic/main.coco | 3 --- tests/src/extras.coco | 4 ++++ 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index c09e349e3..e8881ec9f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,6 @@ python: - '2.7' - pypy - '3.3' -- '3.4' - '3.5' - '3.6' - pypy3 diff --git a/DOCS.md b/DOCS.md index 8eb1d7a9c..a66d96e4a 100644 --- a/DOCS.md +++ b/DOCS.md @@ -184,9 +184,7 @@ By default, if the `source` argument to the command-line utility is a file, it w ### Compatible Python Versions -While Coconut syntax is based off of Python 3, Coconut code compiled in universal mode (the default `--target`), and the Coconut compiler, should run on any Python version `>= 2.6` on the `2.x` branch or `>= 3.2` on the `3.x` branch. - -_Note: The tested against implementations are [CPython](https://www.python.org/) `2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6` and [PyPy](http://pypy.org/) `2.7, 3.2`._ +While Coconut syntax is based off of Python 3, Coconut code compiled in universal mode (the default `--target`), and the Coconut compiler, should run on any Python version `>= 2.6` on the `2.x` branch or `>= 3.2` on the `3.x` branch on either [CPython](https://www.python.org/) or [PyPy](http://pypy.org/). To make Coconut built-ins universal across Python versions, **Coconut automatically overwrites Python 2 built-ins with their Python 3 counterparts**. Additionally, Coconut also overwrites some Python 3 built-ins for optimization and enhancement purposes. If access to the original Python versions of any overwritten built-ins is desired, the old built-ins can be retrieved by prefixing them with `py_`. diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index ea3446622..737053e9a 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -1,8 +1,6 @@ import sys import asyncio # type: ignore -import numpy as np - def main_test(): """Basic no-dependency tests.""" @@ -513,7 +511,6 @@ def main_test(): assert f"{1, 2}" == "(1, 2)" match {"a": {"b": x }} or {"a": {"b": {"c": x}}} = {"a": {"b": {"c": "x"}}} assert x == {"c": "x"} - assert np.all(fmap(-> _ + 1, np.arange(3)) == np.array([1, 2, 3])) assert py_repr("x") == ("u'x'" if sys.version_info < (3,) else "'x'") def foo(x is int) = x try: diff --git a/tests/src/extras.coco b/tests/src/extras.coco index 1c62a3f09..609229289 100644 --- a/tests/src/extras.coco +++ b/tests/src/extras.coco @@ -7,6 +7,7 @@ from coconut.constants import ( IPY, PY35, WINDOWS, + PYPY, ) # type: ignore from coconut.exceptions import ( CoconutSyntaxError, @@ -150,6 +151,9 @@ def test_extras(): assert keyword_complete_result["cursor_start"] == 0 assert keyword_complete_result["cursor_end"] == 1 assert not keyword_complete_result["metadata"] + if not PYPY: + import numpy as np + assert np.all(fmap(-> _ + 1, np.arange(3)) == np.array([1, 2, 3])) print("") if __name__ == "__main__": From 402ec2ae6a0cb4e0a16f015feb6d0b5c7c531760 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 6 Mar 2019 00:16:38 -0800 Subject: [PATCH 076/163] Fix exit codes --- coconut/command/command.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index 43af3f914..41742c599 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -275,7 +275,7 @@ def register_error(self, code=1, errmsg=None): elif errmsg not in self.errmsg: self.errmsg += ", " + errmsg if code is not None: - self.exit_code = max(self.exit_code, code) + self.exit_code = code or self.exit_code @contextmanager def handling_exceptions(self): @@ -569,7 +569,7 @@ def execute(self, compiled=None, path=None, use_eval=False, allow_show=True): print(compiled) if path is not None: # path means header is included, and thus encoding must be removed compiled = rem_encoding(compiled) - self.runner.run(compiled, use_eval=use_eval, path=path, all_errors_exit=(path is not None)) + self.runner.run(compiled, use_eval=use_eval, path=path, all_errors_exit=path is not None) self.run_mypy(code=self.runner.was_run_code()) def execute_file(self, destpath): From 5407fa3a9749a5a5820bff12a9342d78d7d2b931 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 6 Mar 2019 01:06:19 -0800 Subject: [PATCH 077/163] Fix pypy3 test --- tests/main_test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/main_test.py b/tests/main_test.py index a32749840..845db202b 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -467,11 +467,12 @@ def test_pyprover(self): comp_pyprover() run_pyprover() - def test_prelude(self): - with remove_when_done(prelude): - comp_prelude() - if PY35: # has typing - run_prelude() + if not PYPY or PY2: + def test_prelude(self): + with remove_when_done(prelude): + comp_prelude() + if PY35: # has typing + run_prelude() def test_pyston(self): with remove_when_done(pyston): From 6c1a16d5cac0f955580635138f21f5f66a81baa6 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 7 Mar 2019 13:58:30 -0800 Subject: [PATCH 078/163] Add match data types Resolves #494. --- DOCS.md | 30 ++++++++++ coconut/compiler/compiler.py | 81 ++++++++++++++++++++++---- coconut/compiler/grammar.py | 10 +++- coconut/compiler/matching.py | 16 +++-- coconut/root.py | 2 +- tests/src/cocotest/agnostic/suite.coco | 21 +++++++ tests/src/cocotest/agnostic/util.coco | 21 +++++++ 7 files changed, 164 insertions(+), 17 deletions(-) diff --git a/DOCS.md b/DOCS.md index a66d96e4a..931d7dd71 100644 --- a/DOCS.md +++ b/DOCS.md @@ -998,6 +998,36 @@ def classify_sequence(value): **Python:** _Can't be done without a long series of checks for each `match` statement. See the compiled code for the Python syntax._ +### `match data` + +In addition to normal `data` statements, Coconut also supports pattern-matching data statements that enable the use of Coconut's pattern-matching syntax to define the data type's constructor. Pattern-matching data types look like +``` +[match] data () [from ]: + +``` +where `` are exactly as in [pattern-matching functions](#pattern-matching-functions). + +It is important to keep in mind that pattern-matching data types vary from normal data types in a variety of ways. First, like pattern-matching functions, they raise [`MatchError`](#matcherror) instead of `TypeError` when passed the wrong arguments. Second, pattern-matching data types will not do any special handling of starred arguments. Thus, +``` +data vec(*xs) +``` +when iterated over will iterate over all the elements of `xs`, but +``` +match data vec(*xs) +``` +when iterated over will only give the single element `xs`. + +##### Example + +**Coconut:** +``` +data namedpt(name is str, x is int, y is int): + def mag(self) = (self.x**2 + self.y**2)**0.5 +``` + +**Python:** +_Can't be done without a series of method definitions for each data type. See the compiled code for the Python syntax._ + ### `where` Coconut's `where` statement is extremely straightforward. The syntax for a `where` statement is just diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 6624a6d7f..eb07718af 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -422,6 +422,7 @@ def bind(self): self.return_typedef <<= trace(attach(self.return_typedef_ref, self.typedef_handle)) self.typed_assign_stmt <<= trace(attach(self.typed_assign_stmt_ref, self.typed_assign_stmt_handle)) self.datadef <<= trace(attach(self.datadef_ref, self.data_handle)) + self.match_datadef <<= trace(attach(self.match_datadef_ref, self.match_data_handle)) self.with_stmt <<= trace(attach(self.with_stmt_ref, self.with_stmt_handle)) self.await_item <<= trace(attach(self.await_item_ref, self.await_item_handle)) self.ellipsis <<= trace(attach(self.ellipsis_ref, self.ellipsis_handle)) @@ -1214,6 +1215,55 @@ def classlist_handle(self, original, loc, tokens): else: raise CoconutInternalException("invalid classlist tokens", tokens) + def match_data_handle(self, original, loc, tokens): + """Process pattern-matching data blocks.""" + if len(tokens) == 3: + name, match_tokens, stmts = tokens + inherit = None + elif len(tokens) == 4: + name, match_tokens, inherit, stmts = tokens + else: + raise CoconutInternalException("invalid pattern-matching data tokens", tokens) + + if len(match_tokens) == 1: + matches, = match_tokens + cond = None + elif len(match_tokens) == 2: + matches, cond = match_tokens + else: + raise CoconutInternalException("invalid pattern-matching tokens in data", match_tokens) + + matcher = Matcher(loc, match_check_var, name_list=[]) + + req_args, def_args, star_arg, kwd_args, dubstar_arg = split_args_list(matches, loc) + matcher.match_function(match_to_args_var, match_to_kwargs_var, req_args + def_args, star_arg, kwd_args, dubstar_arg) + + if cond is not None: + matcher.add_guard(cond) + + arg_names = ", ".join(matcher.name_list) + arg_tuple = arg_names + ("," if len(matcher.name_list) == 1 else "") + + extra_stmts = '''def __new__(_cls, *{match_to_args_var}, **{match_to_kwargs_var}): + {oind}{match_check_var} = False + {matching} + {pattern_error} + return _coconut.tuple.__new__(_cls, ({arg_tuple})) + {cind}'''.format( + oind=openindent, + cind=closeindent, + match_to_args_var=match_to_args_var, + match_to_kwargs_var=match_to_kwargs_var, + match_check_var=match_check_var, + matching=matcher.out(), + pattern_error=self.pattern_error(original, loc, match_to_args_var, match_check_var, function_match_error_var), + arg_tuple=arg_tuple, + ) + + namedtuple_call = '_coconut.collections.namedtuple("' + name + '", "' + arg_names + '")' + + return self.assemble_data(name, namedtuple_call, inherit, extra_stmts, stmts) + def data_handle(self, loc, tokens): """Process data blocks.""" if len(tokens) == 3: @@ -1272,16 +1322,7 @@ def data_handle(self, loc, tokens): all_args.append(arg_str) attr_str = " ".join(base_args) - extra_stmts = '''__slots__ = () -__ne__ = _coconut.object.__ne__ -def __eq__(self, other): - {oind}return self.__class__ is other.__class__ and _coconut.tuple.__eq__(self, other) -{cind}def __hash__(self): - {oind}return _coconut.tuple.__hash__(self) ^ hash(self.__class__) -{cind}'''.format( - oind=openindent, - cind=closeindent, - ) + extra_stmts = "" if starred_arg is not None: attr_str += (" " if attr_str else "") + starred_arg if base_args: @@ -1361,6 +1402,10 @@ def {arg}(self): else: namedtuple_call = '_coconut.collections.namedtuple("' + name + '", "' + attr_str + '")' + return self.assemble_data(name, namedtuple_call, inherit, extra_stmts, stmts) + + def assemble_data(self, name, namedtuple_call, inherit, extra_stmts, stmts): + # create class out = ( "class " + name + "(" + namedtuple_call + ( ", " + inherit if inherit is not None @@ -1368,6 +1413,20 @@ def {arg}(self): else "" ) + "):\n" + openindent ) + + # add universal statements + extra_stmts = '''__slots__ = () + __ne__ = _coconut.object.__ne__ + def __eq__(self, other): + {oind}return self.__class__ is other.__class__ and _coconut.tuple.__eq__(self, other) + {cind}def __hash__(self): + {oind}return _coconut.tuple.__hash__(self) ^ hash(self.__class__) + {cind}'''.format( + oind=openindent, + cind=closeindent, + ) + extra_stmts + + # manage docstring rest = None if "simple" in stmts and len(stmts) == 1: out += extra_stmts @@ -1384,6 +1443,8 @@ def {arg}(self): out += extra_stmts.rstrip() + stmts[0] else: raise CoconutInternalException("invalid inner data tokens", stmts) + + # create full data definition if rest is not None and rest != "pass\n": out += rest out += closeindent diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 99acb99a8..f9a1c7199 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -1603,7 +1603,13 @@ class Grammar(object): | (newline.suppress() + indent.suppress() + docstring + dedent.suppress() | docstring)("docstring") | simple_stmt("simple") ) | newline("empty")) - datadef_ref = keyword("data").suppress() + name - data_args - data_suite + datadef_ref = keyword("data").suppress() + name + data_args + data_suite + + match_datadef = Forward() + match_data_args = lparen.suppress() + Group( + match_args_list + match_guard, + ) + rparen.suppress() + Optional(keyword("from").suppress() + testlist) + match_datadef_ref = Optional(keyword("match").suppress()) + keyword("data").suppress() + name + match_data_args + data_suite simple_decorator = condense(dotted_name + Optional(function_call))("simple") complex_decorator = test("test") @@ -1624,7 +1630,7 @@ class Grammar(object): decoratable_func_stmt = decoratable_normal_funcdef_stmt | decoratable_async_funcdef_stmt - class_stmt = classdef | datadef + class_stmt = classdef | datadef | match_datadef decoratable_class_stmt = trace(condense(Optional(decorators) + class_stmt)) passthrough_stmt = condense(passthrough_block - (base_suite | newline)) diff --git a/coconut/compiler/matching.py b/coconut/compiler/matching.py index b556b717b..e0ea1aea2 100644 --- a/coconut/compiler/matching.py +++ b/coconut/compiler/matching.py @@ -94,11 +94,12 @@ class Matcher(object): "checkdefs", "names", "var_index", + "name_list", "others", "guards", ) - def __init__(self, loc, check_var, checkdefs=None, names=None, var_index=0): + def __init__(self, loc, check_var, checkdefs=None, names=None, var_index=0, name_list=None): """Creates the matcher.""" self.loc = loc self.check_var = check_var @@ -112,6 +113,7 @@ def __init__(self, loc, check_var, checkdefs=None, names=None, var_index=0): self.set_position(-1) self.names = names if names is not None else {} self.var_index = var_index + self.name_list = name_list self.others = [] self.guards = [] @@ -120,11 +122,17 @@ def duplicate(self, separate_names=False): new_names = self.names if separate_names: new_names = new_names.copy() - other = Matcher(self.loc, self.check_var, self.checkdefs, new_names, self.var_index) + other = Matcher(self.loc, self.check_var, self.checkdefs, new_names, self.var_index, self.name_list) other.insert_check(0, "not " + self.check_var) self.others.append(other) return other + def register_name(self, name, value): + """Register a new name.""" + self.names[name] = value + if self.name_list is not None and name not in self.name_list: + self.name_list.append(name) + def add_guard(self, cond): """Adds cond as a guard.""" self.guards.append(cond) @@ -548,7 +556,7 @@ def match_var(self, tokens, item): self.add_check(self.names[setvar] + " == " + item) else: self.add_def(setvar + " = " + item) - self.names[setvar] = item + self.register_name(setvar, item) def match_set(self, tokens, item): """Matches a set.""" @@ -594,7 +602,7 @@ def match_trailer(self, tokens, item): self.add_check(self.names[arg] + " == " + item) elif arg != wildcard: self.add_def(arg + " = " + item) - self.names[arg] = item + self.register_name(arg, item) else: raise CoconutInternalException("invalid trailer match operation", op) self.match(match, item) diff --git a/coconut/root.py b/coconut/root.py index 512b21e92..7b229ca28 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 18 +DEVELOP = 19 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/cocotest/agnostic/suite.coco b/tests/src/cocotest/agnostic/suite.coco index d0fc6f9df..dbfc024c9 100644 --- a/tests/src/cocotest/agnostic/suite.coco +++ b/tests/src/cocotest/agnostic/suite.coco @@ -535,6 +535,27 @@ def suite_test(): f <**..= ret_dict assert f() == ((1,), dict(x=2)) + assert data1(1) |> fmap$(-> _ + 1) == data1(2) + assert data1(1).x == 1 + assert data2(1) |> fmap$(-> _ + 1) == data2(2) + try: + data2("a") + except MatchError as err: + assert err + else: + assert False + assert data3(1, 2, 3).xs == (1, 2, 3) + assert data4(x=1, y=2).kws == dict(x=1, y=2) + assert data5(1, 2, "3").__doc__ == "docstring" + assert data5(1, 2, "3").attr == 1 + try: + data5(1, 2, 3) + except MatchError as err: + assert err + else: + assert False + assert issubclass(data6, BaseClass) + assert namedpt("a", 3, 4).mag() == 5 return True def tco_test(): diff --git a/tests/src/cocotest/agnostic/util.coco b/tests/src/cocotest/agnostic/util.coco index 04046dfb9..0f8dcde1c 100644 --- a/tests/src/cocotest/agnostic/util.coco +++ b/tests/src/cocotest/agnostic/util.coco @@ -882,3 +882,24 @@ addpattern def join_pairs2([(k, v)] + tail) = # type: ignore result = join_pairs2(tail) result[k] += v result + + +# Match data +match data data1(x) + +data data2(x is int) + +match data data3(*xs) + +data data4(**kws) + +data data5(x, y is int, z is str): + """docstring""" + attr = 1 + +class BaseClass + +data data6(x is int) from BaseClass + +data namedpt(name is str, x is int, y is int): + def mag(self) = (self.x**2 + self.y**2)**0.5 From 85dd6994a12fc18fc8641383ac3bb1d1ca53cbf6 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 7 Mar 2019 22:43:39 -0800 Subject: [PATCH 079/163] Stop strict versioning of prompt_toolkit --- coconut/constants.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index f0ddbb19b..8cf99cc36 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -163,7 +163,6 @@ def checksum(data): "jupyter-console": (5, 2), "ipykernel": (4, 10), "mypy": (0, 660), - "prompt_toolkit": (1,), "futures": (3, 2), "backports.functools-lru-cache": (1, 5), "argparse": (1, 4), @@ -176,8 +175,9 @@ def checksum(data): "pytest": (3,), # don't upgrade this; it breaks on unix "vprof": (0, 36), - # we can't upgrade this; it breaks on Python 2 + # we can't upgrade these; they breaks on Python 2 "ipython": (5, 4), + "prompt_toolkit": (1,), # don't upgrade these; they break on master "sphinx": (1, 7, 4), "sphinx_bootstrap_theme": (0, 4), @@ -186,7 +186,6 @@ def checksum(data): version_strictly = ( "pyparsing", "ipython", - "prompt_toolkit", "sphinx", "sphinx_bootstrap_theme", ) From 4d592a4b374572796c381b6f75c60d62b5262307 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 7 Mar 2019 22:47:38 -0800 Subject: [PATCH 080/163] Bump develop version --- coconut/root.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/root.py b/coconut/root.py index 7b229ca28..a955fab63 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 19 +DEVELOP = 20 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 74578f84ce47fef67d6093ff8a5713ce10736de1 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 8 Mar 2019 00:09:36 -0800 Subject: [PATCH 081/163] Fix prompt_toolkit versioning --- coconut/constants.py | 9 +++++++-- coconut/requirements.py | 14 +++++++++++--- coconut/root.py | 2 +- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index 8cf99cc36..3075464e4 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -104,11 +104,14 @@ def checksum(data): ), "non-py26": ( "pygments", - "prompt_toolkit", ), "py2": ( "futures", "backports.functools-lru-cache", + "prompt_toolkit:2", + ), + "py3": ( + "prompt_toolkit:3", ), "py26": ( "argparse", @@ -171,13 +174,14 @@ def checksum(data): "trollius": (2, 2), "requests": (2,), "numpy": (1,), + "prompt_toolkit:3": (2,), # don't upgrade this; it breaks on Python 2.6 "pytest": (3,), # don't upgrade this; it breaks on unix "vprof": (0, 36), # we can't upgrade these; they breaks on Python 2 "ipython": (5, 4), - "prompt_toolkit": (1,), + "prompt_toolkit:2": (1,), # don't upgrade these; they break on master "sphinx": (1, 7, 4), "sphinx_bootstrap_theme": (0, 4), @@ -188,6 +192,7 @@ def checksum(data): "ipython", "sphinx", "sphinx_bootstrap_theme", + "prompt_toolkit:2", ) classifiers = ( diff --git a/coconut/requirements.py b/coconut/requirements.py index d7b4f9542..4fc231d12 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -44,11 +44,16 @@ # ----------------------------------------------------------------------------------------------------------------------- +def get_base_req(req): + """Get the name of the required package for the given requirement.""" + return req.split(":", 1)[0] + + def get_reqs(which="main"): """Gets requirements from all_reqs with versions.""" reqs = [] for req in all_reqs[which]: - req_str = req + ">=" + ver_tuple_to_str(min_versions[req]) + req_str = get_base_req(req) + ">=" + ver_tuple_to_str(min_versions[req]) if req in version_strictly: req_str += ",<" + ver_tuple_to_str(min_versions[req][:-1] + (min_versions[req][-1] + 1,)) reqs.append(req_str) @@ -117,6 +122,7 @@ def everything_in(req_dict): extras[":python_version<'2.7'"] = get_reqs("py26") extras[":python_version>='2.7'"] = get_reqs("non-py26") extras[":python_version<'3'"] = get_reqs("py2") + extras[":python_version>='3'"] = get_reqs("py3") else: # old method for adding version-dependent requirements @@ -126,6 +132,8 @@ def everything_in(req_dict): requirements += get_reqs("non-py26") if PY2: requirements += get_reqs("py2") + else: + requirements += get_reqs("py3") # ----------------------------------------------------------------------------------------------------------------------- # MAIN: @@ -135,7 +143,7 @@ def everything_in(req_dict): def all_versions(req): """Get all versions of req from PyPI.""" import requests - url = "https://pypi.python.org/pypi/" + req + "/json" + url = "https://pypi.python.org/pypi/" + get_base_req(req) + "/json" return tuple(requests.get(url).json()["releases"].keys()) @@ -166,7 +174,7 @@ def print_new_versions(strict=False): new_versions.append(ver_str) elif not strict and newer(ver_str_to_tuple(ver_str), min_versions[req]): same_versions.append(ver_str) - update_str = req + ": " + ver_tuple_to_str(min_versions[req]) + " -> " + ", ".join( + update_str = req + " = " + ver_tuple_to_str(min_versions[req]) + " -> " + ", ".join( new_versions + ["(" + v + ")" for v in same_versions], ) if new_versions: diff --git a/coconut/root.py b/coconut/root.py index a955fab63..afde3462b 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 20 +DEVELOP = 21 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From d98fdbd24a0c01b64c06d4117a6ac787b1acdac1 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 8 Mar 2019 15:50:49 -0800 Subject: [PATCH 082/163] Use env markers --- coconut/constants.py | 8 +++++--- coconut/requirements.py | 21 ++++++++++++++++++++- coconut/root.py | 2 +- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index 3075464e4..bf528302d 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -121,7 +121,8 @@ def checksum(data): ), "jupyter": ( "jupyter", - "ipython", + ("ipython", "py2"), + ("ipython", "py3"), "ipykernel", "jupyter-console", ), @@ -175,12 +176,13 @@ def checksum(data): "requests": (2,), "numpy": (1,), "prompt_toolkit:3": (2,), + ("ipython", "py3"): (6,), # don't upgrade this; it breaks on Python 2.6 "pytest": (3,), # don't upgrade this; it breaks on unix "vprof": (0, 36), # we can't upgrade these; they breaks on Python 2 - "ipython": (5, 4), + ("ipython", "py2"): (5, 4), "prompt_toolkit:2": (1,), # don't upgrade these; they break on master "sphinx": (1, 7, 4), @@ -189,7 +191,7 @@ def checksum(data): version_strictly = ( "pyparsing", - "ipython", + ("ipython", "py2"), "sphinx", "sphinx_bootstrap_theme", "prompt_toolkit:2", diff --git a/coconut/requirements.py b/coconut/requirements.py index 4fc231d12..ad4494153 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -35,9 +35,12 @@ try: import setuptools # this import is expensive, so we keep it out of constants - using_modern_setuptools = int(setuptools.__version__.split(".", 1)[0]) >= 18 + setuptools_version = tuple(int(x) for x in setuptools.__version__.split(".")) + using_modern_setuptools = setuptools_version >= (18,) + supports_env_markers = setuptools_version >= (36, 2) except Exception: using_modern_setuptools = False + supports_env_markers = False # ----------------------------------------------------------------------------------------------------------------------- # UTILITIES: @@ -46,6 +49,8 @@ def get_base_req(req): """Get the name of the required package for the given requirement.""" + if isinstance(req, tuple): + req = req[0] return req.split(":", 1)[0] @@ -56,6 +61,20 @@ def get_reqs(which="main"): req_str = get_base_req(req) + ">=" + ver_tuple_to_str(min_versions[req]) if req in version_strictly: req_str += ",<" + ver_tuple_to_str(min_versions[req][:-1] + (min_versions[req][-1] + 1,)) + env_marker = req[1] if isinstance(req, tuple) else None + if env_marker: + if env_marker == "py2": + if supports_env_markers: + req_str += ";python_version<'3'" + elif not PY2: + continue + elif env_marker == "py3": + if supports_env_markers: + req_str += ";python_version>='3'" + elif PY2: + continue + else: + raise ValueError("unknown env marker id " + repr(env_marker)) reqs.append(req_str) return reqs diff --git a/coconut/root.py b/coconut/root.py index afde3462b..5158c4448 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 21 +DEVELOP = 22 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 996d9f33250c1a5133967bdd943583f58be3c092 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 8 Mar 2019 17:46:16 -0800 Subject: [PATCH 083/163] Update deps --- coconut/constants.py | 20 +++++++++++++------- coconut/requirements.py | 10 ++++++++-- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index bf528302d..4051905fb 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -123,8 +123,10 @@ def checksum(data): "jupyter", ("ipython", "py2"), ("ipython", "py3"), - "ipykernel", - "jupyter-console", + ("ipykernel", "py2"), + ("ipykernel", "py3"), + ("jupyter-console", "py2"), + ("jupyter-console", "py3"), ), "mypy": ( "mypy", @@ -164,9 +166,7 @@ def checksum(data): "recommonmark": (0, 5), "psutil": (5,), "jupyter": (1, 0), - "jupyter-console": (5, 2), - "ipykernel": (4, 10), - "mypy": (0, 660), + "mypy": (0, 670), "futures": (3, 2), "backports.functools-lru-cache": (1, 5), "argparse": (1, 4), @@ -176,13 +176,17 @@ def checksum(data): "requests": (2,), "numpy": (1,), "prompt_toolkit:3": (2,), - ("ipython", "py3"): (6,), + ("ipython", "py3"): (7,), + ("jupyter-console", "py3"): (6,), + ("ipykernel", "py3"): (5,), # don't upgrade this; it breaks on Python 2.6 "pytest": (3,), # don't upgrade this; it breaks on unix "vprof": (0, 36), # we can't upgrade these; they breaks on Python 2 ("ipython", "py2"): (5, 4), + ("jupyter-console", "py2"): (5, 2), + ("ipykernel", "py2"): (4, 10), "prompt_toolkit:2": (1,), # don't upgrade these; they break on master "sphinx": (1, 7, 4), @@ -191,10 +195,12 @@ def checksum(data): version_strictly = ( "pyparsing", + "prompt_toolkit:2", ("ipython", "py2"), + ("jupyter-console", "py2"), + ("ipykernel", "py2"), "sphinx", "sphinx_bootstrap_theme", - "prompt_toolkit:2", ) classifiers = ( diff --git a/coconut/requirements.py b/coconut/requirements.py index ad4494153..3aafd0cb5 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -193,8 +193,14 @@ def print_new_versions(strict=False): new_versions.append(ver_str) elif not strict and newer(ver_str_to_tuple(ver_str), min_versions[req]): same_versions.append(ver_str) - update_str = req + " = " + ver_tuple_to_str(min_versions[req]) + " -> " + ", ".join( - new_versions + ["(" + v + ")" for v in same_versions], + if isinstance(req, tuple): + base_req, env_marker = req + else: + base_req, env_marker = req, None + update_str = ( + base_req + (" (" + env_marker + ")" if env_marker else "") + + " = " + ver_tuple_to_str(min_versions[req]) + + " -> " + ", ".join(new_versions + ["(" + v + ")" for v in same_versions]) ) if new_versions: new_updates.append(update_str) From 8523d6cafbc64f97e1edcd5d8cc3633357b1db66 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 8 Mar 2019 17:53:00 -0800 Subject: [PATCH 084/163] Strictly version mypy --- coconut/constants.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/coconut/constants.py b/coconut/constants.py index 4051905fb..c1262c031 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -98,6 +98,10 @@ def checksum(data): description = "Simple, elegant, Pythonic functional programming." website_url = "http://coconut-lang.org" +# the different categories here are defined in requirements.py, +# anything after a colon is ignored but allows different versions +# for different categories, and tuples denote the use of environment +# markers as specified in requirements.py all_reqs = { "main": ( "pyparsing", @@ -183,7 +187,7 @@ def checksum(data): "pytest": (3,), # don't upgrade this; it breaks on unix "vprof": (0, 36), - # we can't upgrade these; they breaks on Python 2 + # don't upgrade these; they break on Python 2 ("ipython", "py2"): (5, 4), ("jupyter-console", "py2"): (5, 2), ("ipykernel", "py2"): (4, 10), @@ -201,6 +205,7 @@ def checksum(data): ("ipykernel", "py2"), "sphinx", "sphinx_bootstrap_theme", + "mypy", ) classifiers = ( From 898dfabe57ea54653328fbba856f83adfa6b1b94 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 9 Mar 2019 01:12:26 -0800 Subject: [PATCH 085/163] Fix highlighting --- coconut/constants.py | 21 ++++++++++----------- coconut/highlighter.py | 5 ++++- coconut/icoconut/root.py | 3 ++- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index c1262c031..89ceec579 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -327,7 +327,8 @@ def checksum(data): packrat_cache = 512 -default_whitespace_chars = " \t\f\v\xa0" # we don't include \r here because the compiler converts \r into \n +# we don't include \r here because the compiler converts \r into \n +default_whitespace_chars = " \t\f\v\xa0" varchars = string.ascii_letters + string.digits + "_" @@ -474,7 +475,6 @@ def checksum(data): "where", ) - py3_to_py2_stdlib = { # new_name: (old_name, before_version_info) "builtins": ("__builtin__", (3,)), @@ -633,6 +633,7 @@ def checksum(data): "py_str", "py_map", "py_zip", + "py_repr", ) new_operators = ( @@ -641,17 +642,15 @@ def checksum(data): r"\$", r"`", r"::", - r"(?:<\*?)?(?!\.\.\.)\.\.(?:\*?>)?", # .. - r"\|>", - r"<\|", - r"\|\*>", - r"<\*\|", + r"(?:<\*?\*?)?(?!\.\.\.)\.\.(?:\*?\*?>)?", # .. + r"\|\*?\*?>", + r"<\*?\*?\|", r"->", r"\?\??", "\u2192", # -> - "\\*?\u21a6", # |> - "\u21a4\\*?", # <| - "?", # .. + "\\*?\\*?\u21a6", # |> + "\u21a4\\*?\\*?", # <| + "?", # .. "\u22c5", # * "\u2191", # ** "\xf7", # / @@ -677,7 +676,7 @@ def checksum(data): # ICOCONUT CONSTANTS: # ----------------------------------------------------------------------------------------------------------------------- -py_syntax_version = 3.6 +py_syntax_version = 3 mimetype = "text/x-python3" all_keywords = keywords + const_vars + reserved_vars diff --git a/coconut/highlighter.py b/coconut/highlighter.py index c0776f031..09242dd65 100644 --- a/coconut/highlighter.py +++ b/coconut/highlighter.py @@ -80,7 +80,10 @@ class CoconutLexer(Python3Lexer): tokens = Python3Lexer.tokens.copy() tokens["root"] = [ (r"|".join(new_operators), Operator), - (r'(? Date: Sun, 10 Mar 2019 13:07:02 -0700 Subject: [PATCH 086/163] Fix mypy errors --- coconut/constants.py | 4 ++-- coconut/root.py | 2 +- coconut/stubs/__coconut__.pyi | 10 +++++++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index 89ceec579..05aed6182 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -180,9 +180,9 @@ def checksum(data): "requests": (2,), "numpy": (1,), "prompt_toolkit:3": (2,), - ("ipython", "py3"): (7,), + ("ipython", "py3"): (7, 3), ("jupyter-console", "py3"): (6,), - ("ipykernel", "py3"): (5,), + ("ipykernel", "py3"): (5, 1), # don't upgrade this; it breaks on Python 2.6 "pytest": (3,), # don't upgrade this; it breaks on unix diff --git a/coconut/root.py b/coconut/root.py index 5158c4448..d5ded233a 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 22 +DEVELOP = 23 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/coconut/stubs/__coconut__.pyi b/coconut/stubs/__coconut__.pyi index 851d522ad..0a130cb79 100644 --- a/coconut/stubs/__coconut__.pyi +++ b/coconut/stubs/__coconut__.pyi @@ -143,7 +143,7 @@ def _coconut_igetitem( def _coconut_base_compose( func: _t.Callable[[_T], _t.Any], - *funcstars: _t.Tuple[_t.Callable, bool], + *funcstars: _t.Tuple[_t.Callable, int], ) -> _t.Callable[[_T], _t.Any]: ... @@ -167,7 +167,9 @@ def _coconut_forward_compose( ) -> _t.Callable[..., _W]: ... @_t.overload def _coconut_forward_compose(*funcs: _t.Callable) -> _t.Callable: ... + _coconut_forward_star_compose = _coconut_forward_compose +_coconut_forward_dubstar_compose = _coconut_forward_compose @_t.overload @@ -190,7 +192,9 @@ def _coconut_back_compose( ) -> _t.Callable[..., _W]: ... @_t.overload def _coconut_back_compose(*funcs: _t.Callable) -> _t.Callable: ... + _coconut_back_star_compose = _coconut_back_compose +_coconut_back_dubstar_compose = _coconut_back_compose def _coconut_pipe(x: _T, f: _t.Callable[[_T], _U]) -> _U: ... @@ -201,6 +205,10 @@ def _coconut_star_pipe(xs: _t.Iterable, f: _t.Callable[..., _T]) -> _T: ... def _coconut_back_star_pipe(f: _t.Callable[..., _T], xs: _t.Iterable) -> _T: ... +def _coconut_dubstar_pipe(kws: _t.Dict, f: _t.Callable[..., _T]) -> _T: ... +def _coconut_back_dubstar_pipe(f: _t.Callable[..., _T], kws: _t.Dict) -> _T: ... + + def _coconut_bool_and(a, b): return a and b def _coconut_bool_or(a, b): From 8a69bc000e9dfad2bb8703eae0f9b9493132798a Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 10 Mar 2019 13:58:58 -0700 Subject: [PATCH 087/163] Fix stub --- coconut/stubs/__coconut__.pyi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coconut/stubs/__coconut__.pyi b/coconut/stubs/__coconut__.pyi index 0a130cb79..3c55c10b1 100644 --- a/coconut/stubs/__coconut__.pyi +++ b/coconut/stubs/__coconut__.pyi @@ -205,8 +205,8 @@ def _coconut_star_pipe(xs: _t.Iterable, f: _t.Callable[..., _T]) -> _T: ... def _coconut_back_star_pipe(f: _t.Callable[..., _T], xs: _t.Iterable) -> _T: ... -def _coconut_dubstar_pipe(kws: _t.Dict, f: _t.Callable[..., _T]) -> _T: ... -def _coconut_back_dubstar_pipe(f: _t.Callable[..., _T], kws: _t.Dict) -> _T: ... +def _coconut_dubstar_pipe(kws: _t.Dict[_t.Text, _t.Any], f: _t.Callable[..., _T]) -> _T: ... +def _coconut_back_dubstar_pipe(f: _t.Callable[..., _T], kws: _t.Dict[_t.Text, _t.Any]) -> _T: ... def _coconut_bool_and(a, b): From c87db72e5e959d78fc0813380c562e01483d8ee3 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 14 Mar 2019 15:03:02 -0700 Subject: [PATCH 088/163] Improve verbose --- coconut/compiler/compiler.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index eb07718af..c84fbde7c 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -370,11 +370,19 @@ def __reduce__(self): def genhash(self, code, package_level=-1): """Generate a hash from code.""" + reduce_args = self.__reduce__()[1] + logger.log( + "Hash args:", { + "VERSION_STR": VERSION_STR, + "reduce_args": reduce_args, + "package_level": package_level, + }, + ) return hex(checksum( hash_sep.join( str(item) for item in ( (VERSION_STR,) - + self.__reduce__()[1] + + reduce_args + (package_level, code) ) ).encode(default_encoding), From 0a8ee8465ca0fe9cec56166d7a4641893025694b Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 16 Mar 2019 02:28:13 -0700 Subject: [PATCH 089/163] Add --args alias --- DOCS.md | 6 ++++-- coconut/command/cli.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/DOCS.md b/DOCS.md index 931d7dd71..b23bec264 100644 --- a/DOCS.md +++ b/DOCS.md @@ -84,7 +84,8 @@ coconut [-h] [-v] [-t version] [-i] [-p] [-a] [-l] [-k] [-w] [-r] [-n] [-d] [-q] [-s] [--no-tco] [-c code] [-j processes] [-f] [--minify] [--jupyter ...] [--mypy ...] [--argv ...] [--tutorial] [--documentation] [--style name] - [--recursion-limit limit] [--verbose] [--trace] + [--history-file path] [--recursion-limit limit] [--verbose] + [--trace] [source] [dest] ``` @@ -136,7 +137,8 @@ dest destination directory for compiled files (defaults to (remaining args passed to Jupyter) --mypy ... run MyPy on compiled Python (remaining args passed to MyPy) (implies --package) - --argv ... set sys.argv to source plus remaining args for use in + --argv ..., --args ... + set sys.argv to source plus remaining args for use in Coconut script being run --tutorial open Coconut's tutorial in the default web browser --documentation open Coconut's documentation in the default web diff --git a/coconut/command/cli.py b/coconut/command/cli.py index 3c9dfa4bc..2285340ae 100644 --- a/coconut/command/cli.py +++ b/coconut/command/cli.py @@ -185,7 +185,7 @@ ) arguments.add_argument( - "--argv", + "--argv", "--args", type=str, nargs=argparse.REMAINDER, help="set sys.argv to source plus remaining args for use in Coconut script being run", From 52b844852a7b547bfab81ea676a4bc5fffc8a573 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 19 Mar 2019 22:29:54 -0700 Subject: [PATCH 090/163] Add assert operator function Resolves #498. --- DOCS.md | 1 + coconut/compiler/grammar.py | 2 +- coconut/compiler/header.py | 2 +- coconut/compiler/templates/header.py_template | 1 + coconut/root.py | 2 +- coconut/stubs/__coconut__.pyi | 4 ++++ tests/src/cocotest/agnostic/main.coco | 13 +++++++++++++ 7 files changed, 22 insertions(+), 3 deletions(-) diff --git a/DOCS.md b/DOCS.md index b23bec264..386d92e22 100644 --- a/DOCS.md +++ b/DOCS.md @@ -1211,6 +1211,7 @@ A very common thing to do in functional programming is to make use of function v (or) => # boolean or (is) => (operator.is_) (in) => (operator.contains) +(assert) => # assert function ``` ##### Example diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index f9a1c7199..30803949b 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -910,7 +910,6 @@ class Grammar(object): test_expr = yield_expr | testlist_star_expr op_item = ( - # must go dubstar then star then no star fixto(dubstar_pipe, "_coconut_dubstar_pipe") | fixto(back_dubstar_pipe, "_coconut_back_dubstar_pipe") @@ -927,6 +926,7 @@ class Grammar(object): | fixto(comp_pipe, "_coconut_forward_compose") | fixto(dotdot | comp_back_pipe, "_coconut_back_compose") + | fixto(keyword("assert"), "_coconut_assert") | fixto(keyword("and"), "_coconut_bool_and") | fixto(keyword("or"), "_coconut_bool_or") | fixto(dubquestion, "_coconut_none_coalesce") diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 3a7f71ef2..e80b9f587 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -202,7 +202,7 @@ def NamedTuple(name, fields): return _coconut.collections.namedtuple(name, [x for x, t in fields])'''.format(**format_dict), ) - format_dict["underscore_imports"] = "_coconut, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_back_pipe, _coconut_star_pipe, _coconut_back_star_pipe, _coconut_dubstar_pipe, _coconut_back_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_addpattern, _coconut_sentinel".format(**format_dict) + format_dict["underscore_imports"] = "_coconut, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_back_pipe, _coconut_star_pipe, _coconut_back_star_pipe, _coconut_dubstar_pipe, _coconut_back_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_addpattern, _coconut_sentinel, _coconut_assert".format(**format_dict) # ._coconut_tco_func is used in main.coco, so don't remove it # here without replacing its usage there diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 619f5112c..e07ba257a 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -63,6 +63,7 @@ def _coconut_dubstar_pipe(kws, f): return f(**kws) def _coconut_back_pipe(f, x): return f(x) def _coconut_back_star_pipe(f, xs): return f(*xs) def _coconut_back_dubstar_pipe(f, kws): return f(**kws) +def _coconut_assert(cond, msg=None): assert cond, msg if msg is not None else "(assert) got falsey value " + _coconut.repr(cond) def _coconut_bool_and(a, b): return a and b def _coconut_bool_or(a, b): return a or b def _coconut_none_coalesce(a, b): return a if a is not None else b diff --git a/coconut/root.py b/coconut/root.py index d5ded233a..931582c65 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 23 +DEVELOP = 24 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/coconut/stubs/__coconut__.pyi b/coconut/stubs/__coconut__.pyi index 3c55c10b1..559853bcf 100644 --- a/coconut/stubs/__coconut__.pyi +++ b/coconut/stubs/__coconut__.pyi @@ -209,6 +209,10 @@ def _coconut_dubstar_pipe(kws: _t.Dict[_t.Text, _t.Any], f: _t.Callable[..., _T] def _coconut_back_dubstar_pipe(f: _t.Callable[..., _T], kws: _t.Dict[_t.Text, _t.Any]) -> _T: ... +def _coconut_assert(cond, msg: _t.Optional[_t.Text]=None): + assert cond, msg + + def _coconut_bool_and(a, b): return a and b def _coconut_bool_or(a, b): diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index 737053e9a..c4fe1befa 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -517,6 +517,19 @@ def main_test(): foo(["foo"] * 100000) except MatchError as err: assert len(repr(err)) < 1000 + (assert)(True) + try: + (assert)(False, "msg") + except AssertionError as err: + assert str(err) == "msg" + else: + assert False + try: + (assert)([]) + except AssertionError as err: + assert str(err) == "(assert) got falsey value []" + else: + assert False return True def tco_func() = tco_func() From bd192e44e1bb8e422e8d61e76152bea925611f80 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 23 Mar 2019 11:42:01 -0700 Subject: [PATCH 091/163] Fix prompt toolkit versioning Resolves #499. --- coconut/constants.py | 1 - coconut/root.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index 05aed6182..6d94024a2 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -199,7 +199,6 @@ def checksum(data): version_strictly = ( "pyparsing", - "prompt_toolkit:2", ("ipython", "py2"), ("jupyter-console", "py2"), ("ipykernel", "py2"), diff --git a/coconut/root.py b/coconut/root.py index 931582c65..127c15485 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 24 +DEVELOP = 25 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 7c6c18a8a29c9544176f5023ef7d5592b4c5ac42 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 24 Mar 2019 14:39:56 -0700 Subject: [PATCH 092/163] Fix coconut-run --- coconut/command/command.py | 9 +++++---- coconut/constants.py | 3 --- coconut/root.py | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index 41742c599..8f11d0ffc 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -111,17 +111,18 @@ def start(self, run=False): args = list(coconut_run_verbose_args) + args else: args = list(coconut_run_args) + args - args += ["--argv"] + argv + self.cmd(args, argv=argv) else: - args = None - self.cmd(args) + self.cmd() - def cmd(self, args=None, interact=True): + def cmd(self, args=None, argv=None, interact=True): """Process command-line arguments.""" if args is None: parsed_args = arguments.parse_args() else: parsed_args = arguments.parse_args(args) + if argv is not None: + parsed_args.argv = argv self.exit_code = 0 with self.handling_exceptions(): self.use_args(parsed_args, interact, original_args=args) diff --git a/coconut/constants.py b/coconut/constants.py index 6d94024a2..93d0bc569 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -199,9 +199,6 @@ def checksum(data): version_strictly = ( "pyparsing", - ("ipython", "py2"), - ("jupyter-console", "py2"), - ("ipykernel", "py2"), "sphinx", "sphinx_bootstrap_theme", "mypy", diff --git a/coconut/root.py b/coconut/root.py index 127c15485..e77a45470 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 25 +DEVELOP = 26 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 320e8d5feae2892e5b150740004d2fbedf654926 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 15 Apr 2019 14:53:18 -0700 Subject: [PATCH 093/163] Use cPyparsing by default Resolves #501. --- DOCS.md | 4 ++-- FAQ.md | 2 +- coconut/constants.py | 10 +++++----- coconut/requirements.py | 36 ++++++++++++++++++++++-------------- coconut/root.py | 2 +- 5 files changed, 31 insertions(+), 23 deletions(-) diff --git a/DOCS.md b/DOCS.md index 386d92e22..bba089971 100644 --- a/DOCS.md +++ b/DOCS.md @@ -56,13 +56,13 @@ pip install coconut[opt_dep_1,opt_dep_2] The full list of optional dependencies is: -- `all`: alias for `jupyter,watch,jobs,mypy,asyncio,cPyparsing` (this is the recommended way to install a feature-complete version of Coconut), +- `all`: alias for `jupyter,watch,jobs,mypy,asyncio` (this is the recommended way to install a feature-complete version of Coconut), - `jupyter/ipython`: enables use of the `--jupyter` / `--ipython` flag, - `watch`: enables use of the `--watch` flag, - `jobs`: improves use of the `--jobs` flag, - `mypy`: enables use of the `--mypy` flag, - `asyncio`: enables use of the [`asyncio`](https://docs.python.org/3/library/asyncio.html) library on older Python versions by making use of [`trollius`](https://pypi.python.org/pypi/trollius), -- `cPyparsing`: significantly speeds up compilation if your platform supports it by making use of [`cPyparsing`](https://github.com/evhub/cpyparsing), +- `purepython`: uses the pure-Python [`pyparsing`](https://github.com/pyparsing/pyparsing) module instead of the default (faster) [`cPyparsing`](https://github.com/evhub/cpyparsing) module, - `tests`: everything necessary to run Coconut's test suite, - `docs`: everything necessary to build Coconut's documentation, and - `dev`: everything necessary to develop on Coconut, including all of the dependencies above. diff --git a/FAQ.md b/FAQ.md index cd32c7581..2e06ee6ad 100644 --- a/FAQ.md +++ b/FAQ.md @@ -71,7 +71,7 @@ I certainly hope not! Unlike most transpiled languages, all valid Python is vali ### I want to use Coconut in a production environment; how do I achieve maximum performance? -First, you're going to want a fast compiler, so you should either [install Coconut with the `cPyparsing` option](DOCS.html#installation), or use [`PyPy`](https://pypy.org/). Second, there are two simple things you can do to make Coconut produce faster Python: compile with `--no-tco` and compile with a `--target` specification for the exact version of Python you want to run your code on. Passing `--target` helps Coconut optimize the compiled code for the Python version you want, and, though [Tail Call Optimization](DOCS.html#tail-call-optimization) is useful, it will usually significantly slow down functions that use it, so disabling it will often provide a major performance boost. +First, you're going to want a fast compiler, so you should either use [`cPyparsing`](https://github.com/evhub/cpyparsing) or [`PyPy`](https://pypy.org/). Second, there are two simple things you can do to make Coconut produce faster Python: compile with `--no-tco` and compile with a `--target` specification for the exact version of Python you want to run your code on. Passing `--target` helps Coconut optimize the compiled code for the Python version you want, and, though [Tail Call Optimization](DOCS.html#tail-call-optimization) is useful, it will usually significantly slow down functions that use it, so disabling it will often provide a major performance boost. ### I want to contribute to Coconut, how do I get started? diff --git a/coconut/constants.py b/coconut/constants.py index 93d0bc569..c0a77b235 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -103,7 +103,10 @@ def checksum(data): # for different categories, and tuples denote the use of environment # markers as specified in requirements.py all_reqs = { - "main": ( + "cpython": ( + "cPyparsing", + ), + "purepython": ( "pyparsing", ), "non-py26": ( @@ -157,9 +160,6 @@ def checksum(data): "pexpect", "numpy", ), - "cPyparsing": ( - "cPyparsing", - ), } min_versions = { @@ -170,7 +170,7 @@ def checksum(data): "recommonmark": (0, 5), "psutil": (5,), "jupyter": (1, 0), - "mypy": (0, 670), + "mypy": (0, 700), "futures": (3, 2), "backports.functools-lru-cache": (1, 5), "argparse": (1, 4), diff --git a/coconut/requirements.py b/coconut/requirements.py index 3aafd0cb5..ed63e1690 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -17,6 +17,8 @@ from coconut.root import * # NOQA +import platform + from coconut.constants import ( ver_str_to_tuple, ver_tuple_to_str, @@ -54,7 +56,7 @@ def get_base_req(req): return req.split(":", 1)[0] -def get_reqs(which="main"): +def get_reqs(which): """Gets requirements from all_reqs with versions.""" reqs = [] for req in all_reqs[which]: @@ -106,7 +108,7 @@ def everything_in(req_dict): # SETUP: # ----------------------------------------------------------------------------------------------------------------------- -requirements = get_reqs() +requirements = [] extras = { "jupyter": get_reqs("jupyter"), @@ -114,22 +116,22 @@ def everything_in(req_dict): "jobs": get_reqs("jobs"), "mypy": get_reqs("mypy"), "asyncio": get_reqs("asyncio"), - "cPyparsing": get_reqs("cPyparsing"), } extras["all"] = everything_in(extras) -extras["ipython"] = extras["jupyter"] - -extras["docs"] = unique_wrt(get_reqs("docs"), requirements) - -extras["tests"] = uniqueify_all( - get_reqs("tests"), - extras["jobs"] + get_reqs("cPyparsing") if not PYPY else [], - extras["jupyter"] if IPY else [], - extras["mypy"] if PY34 and not WINDOWS and not PYPY else [], - extras["asyncio"] if not PY34 else [], -) +extras.update({ + "ipython": extras["jupyter"], + "purepython": get_reqs("purepython"), + "docs": unique_wrt(get_reqs("docs"), requirements), + "tests": uniqueify_all( + get_reqs("tests"), + extras["jobs"] if not PYPY else [], + extras["jupyter"] if IPY else [], + extras["mypy"] if PY34 and not WINDOWS and not PYPY else [], + extras["asyncio"] if not PY34 else [], + ), +}) extras["dev"] = uniqueify_all( everything_in(extras), @@ -138,6 +140,8 @@ def everything_in(req_dict): if using_modern_setuptools: # modern method for adding version-dependent requirements + extras[":platform_python_implementation=='CPython'"] = get_reqs("cpython") + extras[":platform_python_implementation!='CPython'"] = get_reqs("purepython") extras[":python_version<'2.7'"] = get_reqs("py26") extras[":python_version>='2.7'"] = get_reqs("non-py26") extras[":python_version<'3'"] = get_reqs("py2") @@ -145,6 +149,10 @@ def everything_in(req_dict): else: # old method for adding version-dependent requirements + if platform.python_implementation() == "CPython": + requirements += get_reqs("cpython") + else: + requirements += get_reqs("purepython") if PY26: requirements += get_reqs("py26") else: diff --git a/coconut/root.py b/coconut/root.py index e77a45470..80c8d40c2 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 26 +DEVELOP = 27 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 410c196046ada05c3b1599b74b1f11c15020e52c Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 15 Apr 2019 20:26:29 -0700 Subject: [PATCH 094/163] Update to new pyparsing --- coconut/{myparsing.py => _pyparsing.py} | 30 ++++++++++++++++++++----- coconut/command/command.py | 4 ++-- coconut/compiler/compiler.py | 2 +- coconut/compiler/grammar.py | 2 +- coconut/compiler/util.py | 2 +- coconut/constants.py | 2 +- coconut/exceptions.py | 2 +- coconut/root.py | 2 +- coconut/terminal.py | 2 +- 9 files changed, 34 insertions(+), 14 deletions(-) rename coconut/{myparsing.py => _pyparsing.py} (67%) diff --git a/coconut/myparsing.py b/coconut/_pyparsing.py similarity index 67% rename from coconut/myparsing.py rename to coconut/_pyparsing.py index b8f2a8948..032a2e95c 100644 --- a/coconut/myparsing.py +++ b/coconut/_pyparsing.py @@ -32,28 +32,35 @@ # warning: do not name this file cPyparsing or pyparsing or it might collide with the following imports try: + + import cPyparsing as _pyparsing from cPyparsing import * # NOQA from cPyparsing import ( # NOQA _trim_arity, _ParseResultsWithOffset, __version__, ) - PYPARSING = "Cython cPyparsing v" + __version__ + PYPARSING_PACKAGE = "cPyparsing" + PYPARSING_INFO = "Cython cPyparsing v" + __version__ except ImportError: try: + import pyparsing as _pyparsing from pyparsing import * # NOQA from pyparsing import ( # NOQA _trim_arity, _ParseResultsWithOffset, __version__, ) - PYPARSING = "Python pyparsing v" + __version__ + PYPARSING_PACKAGE = "pyparsing" + PYPARSING_INFO = "Python pyparsing v" + __version__ except ImportError: traceback.print_exc() __version__ = None + PYPARSING_PACKAGE = "pyparsing" + PYPARSING_INFO = None # ----------------------------------------------------------------------------------------------------------------------- # SETUP: @@ -63,9 +70,8 @@ req_ver_str = ver_tuple_to_str(min_versions["pyparsing"]) raise ImportError( "Coconut requires pyparsing version >= " + req_ver_str - + ("; got version " + __version__ if __version__ is not None else "") - + " (run 'pip install --upgrade pyparsing' or" - + " 'pip install --upgrade cPyparsing' to fix)", + + ("; got " + PYPARSING_INFO if PYPARSING_INFO is not None else "") + + " (run 'pip install --upgrade " + PYPARSING_PACKAGE + "' to fix)", ) if packrat_cache: @@ -74,3 +80,17 @@ ParserElement.setDefaultWhitespaceChars(default_whitespace_chars) Keyword.setDefaultKeywordChars(varchars) + +# ----------------------------------------------------------------------------------------------------------------------- +# __str__ Patching: +# ----------------------------------------------------------------------------------------------------------------------- + +# makes pyparsing much faster if it doesn't have to compute expensive +# nested string representations +for obj in vars(_pyparsing).values(): + try: + if issubclass(obj, ParserElement): + obj.__str__ = object.__str__ + obj.__repr__ = object.__repr__ + except TypeError: + pass diff --git a/coconut/command/command.py b/coconut/command/command.py index 8f11d0ffc..8d3424e56 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -27,7 +27,7 @@ from contextlib import contextmanager from subprocess import CalledProcessError -from coconut.myparsing import PYPARSING +from coconut._pyparsing import PYPARSING_INFO from coconut.compiler import Compiler from coconut.exceptions import ( CoconutException, @@ -151,7 +151,7 @@ def use_args(self, args, interact=True, original_args=None): if DEVELOP: logger.tracing = args.trace - logger.log("Using " + PYPARSING + ".") + logger.log("Using " + PYPARSING_INFO + ".") if original_args is not None: logger.log("Directly passed args:", original_args) logger.log("Parsed args:", args) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index c84fbde7c..d389426ab 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -32,7 +32,7 @@ from contextlib import contextmanager from functools import partial -from coconut.myparsing import ( +from coconut._pyparsing import ( ParseBaseException, col, line as getline, diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 30803949b..dc6cc0b59 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -30,7 +30,7 @@ import re from functools import reduce -from coconut.myparsing import ( +from coconut._pyparsing import ( CaselessLiteral, Forward, Group, diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index e84d63df1..7c9f84b8c 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -25,7 +25,7 @@ from functools import partial from contextlib import contextmanager -from coconut.myparsing import ( +from coconut._pyparsing import ( replaceWith, ZeroOrMore, Optional, diff --git a/coconut/constants.py b/coconut/constants.py index c0a77b235..da736be9c 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -163,7 +163,7 @@ def checksum(data): } min_versions = { - "pyparsing": (2, 3, 1), + "pyparsing": (2, 4, 0), "cPyparsing": (2, 3, 2, 1, 0, 0), "pre-commit": (1,), "pygments": (2, 3), diff --git a/coconut/exceptions.py b/coconut/exceptions.py index 08c79f076..d9f7acd25 100644 --- a/coconut/exceptions.py +++ b/coconut/exceptions.py @@ -21,7 +21,7 @@ import sys -from coconut.myparsing import lineno +from coconut._pyparsing import lineno from coconut.constants import ( openindent, diff --git a/coconut/root.py b/coconut/root.py index 80c8d40c2..684978792 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 27 +DEVELOP = 28 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/coconut/terminal.py b/coconut/terminal.py index 602b8bf9a..16c25ebcf 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -25,7 +25,7 @@ import time from contextlib import contextmanager -from coconut.myparsing import ( +from coconut._pyparsing import ( lineno, col, ParserElement, From 8b6ccfe3c39af98cd882d736124bb406232e729a Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 15 Apr 2019 20:28:00 -0700 Subject: [PATCH 095/163] Reorder pyparsing patches --- coconut/_pyparsing.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 032a2e95c..426e2eafb 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -74,17 +74,6 @@ + " (run 'pip install --upgrade " + PYPARSING_PACKAGE + "' to fix)", ) -if packrat_cache: - ParserElement.enablePackrat(packrat_cache) - -ParserElement.setDefaultWhitespaceChars(default_whitespace_chars) - -Keyword.setDefaultKeywordChars(varchars) - -# ----------------------------------------------------------------------------------------------------------------------- -# __str__ Patching: -# ----------------------------------------------------------------------------------------------------------------------- - # makes pyparsing much faster if it doesn't have to compute expensive # nested string representations for obj in vars(_pyparsing).values(): @@ -94,3 +83,10 @@ obj.__repr__ = object.__repr__ except TypeError: pass + +if packrat_cache: + ParserElement.enablePackrat(packrat_cache) + +ParserElement.setDefaultWhitespaceChars(default_whitespace_chars) + +Keyword.setDefaultKeywordChars(varchars) From 7416f5b43364668ba78ea14a437f4d3fe2f910ea Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 15 Apr 2019 20:46:57 -0700 Subject: [PATCH 096/163] Improve pyparsing version checking --- coconut/_pyparsing.py | 11 ++++++++--- coconut/constants.py | 5 +++++ coconut/requirements.py | 3 ++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 426e2eafb..0dd72d75c 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -28,6 +28,7 @@ min_versions, ver_str_to_tuple, ver_tuple_to_str, + get_next_version, ) # warning: do not name this file cPyparsing or pyparsing or it might collide with the following imports @@ -66,10 +67,14 @@ # SETUP: # ----------------------------------------------------------------------------------------------------------------------- -if __version__ is None or ver_str_to_tuple(__version__) < min_versions["pyparsing"]: - req_ver_str = ver_tuple_to_str(min_versions["pyparsing"]) +min_ver = min(min_versions["pyparsing"], min_versions["cPyparsing"][:3]) +max_ver = get_next_version(max(min_versions["pyparsing"], min_versions["cPyparsing"][:3])) + +if __version__ is None or not min_ver <= ver_str_to_tuple(__version__) < max_ver: + min_ver_str = ver_tuple_to_str(min_ver) + max_ver_str = ver_tuple_to_str(max_ver) raise ImportError( - "Coconut requires pyparsing version >= " + req_ver_str + "Coconut requires pyparsing/cPyparsing version >= " + min_ver_str + " and < " + max_ver_str + ("; got " + PYPARSING_INFO if PYPARSING_INFO is not None else "") + " (run 'pip install --upgrade " + PYPARSING_PACKAGE + "' to fix)", ) diff --git a/coconut/constants.py b/coconut/constants.py index da736be9c..40caad748 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -57,6 +57,11 @@ def ver_str_to_tuple(ver_str): return tuple(out) +def get_next_version(req_ver): + """Get the next version after the given version.""" + return req_ver[:-1] + (req_ver[-1] + 1,) + + def checksum(data): """Compute a checksum of the given data. Used for computing __coconut_hash__.""" diff --git a/coconut/requirements.py b/coconut/requirements.py index ed63e1690..4a8ed7e19 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -22,6 +22,7 @@ from coconut.constants import ( ver_str_to_tuple, ver_tuple_to_str, + get_next_version, all_reqs, min_versions, version_strictly, @@ -62,7 +63,7 @@ def get_reqs(which): for req in all_reqs[which]: req_str = get_base_req(req) + ">=" + ver_tuple_to_str(min_versions[req]) if req in version_strictly: - req_str += ",<" + ver_tuple_to_str(min_versions[req][:-1] + (min_versions[req][-1] + 1,)) + req_str += ",<" + ver_tuple_to_str(get_next_version(min_versions[req])) env_marker = req[1] if isinstance(req, tuple) else None if env_marker: if env_marker == "py2": From c49b29ec7ce2aa7909e062721639550aaa03665c Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 15 Apr 2019 20:55:17 -0700 Subject: [PATCH 097/163] Add license to setup.py --- coconut/constants.py | 2 ++ setup.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/coconut/constants.py b/coconut/constants.py index 40caad748..26ffea671 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -103,6 +103,8 @@ def checksum(data): description = "Simple, elegant, Pythonic functional programming." website_url = "http://coconut-lang.org" +license_name = "Apache 2.0" + # the different categories here are defined in requirements.py, # anything after a colon is ignored but allows different versions # for different categories, and tuples denote the use of environment diff --git a/setup.py b/setup.py index 94e977f7e..a8badc36b 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ classifiers, search_terms, script_names, + license_name, ) from coconut.requirements import ( using_modern_setuptools, @@ -83,4 +84,5 @@ }, classifiers=list(classifiers), keywords=list(search_terms), + license=license_name, ) From e0d6426dad21cb93dc1a459487bde0bf33464e9d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 15 Apr 2019 22:44:28 -0700 Subject: [PATCH 098/163] Improve cPyparsing support --- DOCS.md | 9 +++++++-- HELP.md | 2 +- coconut/constants.py | 2 +- coconut/requirements.py | 2 +- coconut/root.py | 2 +- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/DOCS.md b/DOCS.md index bba089971..5e9d85950 100644 --- a/DOCS.md +++ b/DOCS.md @@ -32,7 +32,13 @@ which will install Coconut and its required dependencies. _Note: If you have an old version of Coconut installed and you want to upgrade, run `pip install --upgrade coconut` instead._ -If you are encountering errors running `pip install coconut`, try re-running it with the `--user` option. If `pip install coconut` works, but you cannot access the `coconut` command, be sure that Coconut's installation location is in your `PATH` environment variable. On UNIX, that is `/usr/local/bin` (without `--user`) or `${HOME}/.local/bin/` (with `--user`). +If you are encountering errors running `pip install coconut`, try instead running +``` +pip install --user --no-deps --upgrade coconut pyparsing +``` +which will force Coconut to use the pure-Python [`pyparsing`](https://github.com/pyparsing/pyparsing) module instead of the faster [`cPyparsing`](https://github.com/evhub/cpyparsing) module. If you are still getting errors, you may want to try [using conda](#using-conda) instead. + +If `pip install coconut` works, but you cannot access the `coconut` command, be sure that Coconut's installation location is in your `PATH` environment variable. On UNIX, that is `/usr/local/bin` (without `--user`) or `${HOME}/.local/bin/` (with `--user`). ### Using Conda @@ -62,7 +68,6 @@ The full list of optional dependencies is: - `jobs`: improves use of the `--jobs` flag, - `mypy`: enables use of the `--mypy` flag, - `asyncio`: enables use of the [`asyncio`](https://docs.python.org/3/library/asyncio.html) library on older Python versions by making use of [`trollius`](https://pypi.python.org/pypi/trollius), -- `purepython`: uses the pure-Python [`pyparsing`](https://github.com/pyparsing/pyparsing) module instead of the default (faster) [`cPyparsing`](https://github.com/evhub/cpyparsing) module, - `tests`: everything necessary to run Coconut's test suite, - `docs`: everything necessary to build Coconut's documentation, and - `dev`: everything necessary to develop on Coconut, including all of the dependencies above. diff --git a/HELP.md b/HELP.md index a7be52f71..47ceb1dd9 100644 --- a/HELP.md +++ b/HELP.md @@ -43,7 +43,7 @@ Installing Coconut, including all the features above, is drop-dead simple. Just pip install coconut ``` -_Note: Try re-running the above command with the `--user` option if you are encountering errors. Be sure that Coconut's installation location (on UNIX `/usr/local/bin` if you didn't use `--user` or `${HOME}/.local/bin/` if you did) is in your `PATH` environment variable. If you are still encountering errors installing Coconut with `pip`, you can also install Coconut with `conda` by following the [conda installation instructions in the documentation](DOCS.html#using-conda)._ +_Note: If you are having trouble installing Coconut, try following the debugging steps in the [installation section of Coconut's documentation](DOCS.html#installation)._ To check that your installation is functioning properly, try entering into the command line ``` diff --git a/coconut/constants.py b/coconut/constants.py index 26ffea671..fc9157b92 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -171,7 +171,7 @@ def checksum(data): min_versions = { "pyparsing": (2, 4, 0), - "cPyparsing": (2, 3, 2, 1, 0, 0), + "cPyparsing": (2, 4, 0, 1, 0, 0), "pre-commit": (1,), "pygments": (2, 3), "recommonmark": (0, 5), diff --git a/coconut/requirements.py b/coconut/requirements.py index 4a8ed7e19..aca3c2e9c 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -123,7 +123,6 @@ def everything_in(req_dict): extras.update({ "ipython": extras["jupyter"], - "purepython": get_reqs("purepython"), "docs": unique_wrt(get_reqs("docs"), requirements), "tests": uniqueify_all( get_reqs("tests"), @@ -136,6 +135,7 @@ def everything_in(req_dict): extras["dev"] = uniqueify_all( everything_in(extras), + get_reqs("purepython"), get_reqs("dev"), ) diff --git a/coconut/root.py b/coconut/root.py index 684978792..e14662161 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 28 +DEVELOP = 29 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 7703f74c9f020606f767ff1e9f6381bc732d4ff4 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 16 Apr 2019 01:44:04 -0700 Subject: [PATCH 099/163] Improve prompt_toolkit versioning --- coconut/constants.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/coconut/constants.py b/coconut/constants.py index fc9157b92..dbd796a2c 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -186,7 +186,7 @@ def checksum(data): "trollius": (2, 2), "requests": (2,), "numpy": (1,), - "prompt_toolkit:3": (2,), + "prompt_toolkit:3": (1,), ("ipython", "py3"): (7, 3), ("jupyter-console", "py3"): (6,), ("ipykernel", "py3"): (5, 1), @@ -209,6 +209,7 @@ def checksum(data): "sphinx", "sphinx_bootstrap_theme", "mypy", + "prompt_toolkit:2", ) classifiers = ( From c10a74c7758883930664ad9de3a2ce694bcc4318 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 16 Apr 2019 01:46:30 -0700 Subject: [PATCH 100/163] Always test with latest pip --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c48e79791..230fe89bf 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,11 @@ .PHONY: install install: - pip install "pip>=7.1.2" + pip install --upgrade setuptools pip pip install .[tests] .PHONY: dev dev: - python -m pip install --upgrade setuptools pip + pip install --upgrade setuptools pip pip install --upgrade -e .[dev] pre-commit install -f --install-hooks From 8bf9313838203841664ad0e17b7b4478759e92db Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 16 Apr 2019 01:48:43 -0700 Subject: [PATCH 101/163] Improve setuptools version handling --- coconut/requirements.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/coconut/requirements.py b/coconut/requirements.py index aca3c2e9c..c72d14de1 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -139,21 +139,26 @@ def everything_in(req_dict): get_reqs("dev"), ) -if using_modern_setuptools: - # modern method for adding version-dependent requirements +if supports_env_markers: + # modern method extras[":platform_python_implementation=='CPython'"] = get_reqs("cpython") extras[":platform_python_implementation!='CPython'"] = get_reqs("purepython") +else: + # old method + if platform.python_implementation() == "CPython": + requirements += get_reqs("cpython") + else: + requirements += get_reqs("purepython") + +if using_modern_setuptools: + # modern method extras[":python_version<'2.7'"] = get_reqs("py26") extras[":python_version>='2.7'"] = get_reqs("non-py26") extras[":python_version<'3'"] = get_reqs("py2") extras[":python_version>='3'"] = get_reqs("py3") else: - # old method for adding version-dependent requirements - if platform.python_implementation() == "CPython": - requirements += get_reqs("cpython") - else: - requirements += get_reqs("purepython") + # old method if PY26: requirements += get_reqs("py26") else: From f338252d6038c697672b8773741eebe1b58e73ee Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 16 Apr 2019 15:32:23 -0700 Subject: [PATCH 102/163] Fix __str__ overriding --- coconut/_pyparsing.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 0dd72d75c..fa3c14f1d 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -20,6 +20,7 @@ from coconut.root import * # NOQA import traceback +import functools from coconut.constants import ( packrat_cache, @@ -79,13 +80,24 @@ + " (run 'pip install --upgrade " + PYPARSING_PACKAGE + "' to fix)", ) + +def fast_str(cls): + """A very simple __str__ implementation.""" + return "<" + cls.__name__ + ">" + + +def fast_repr(cls): + """A very simple __repr__ implementation.""" + return "<" + cls.__name__ + ">" + + # makes pyparsing much faster if it doesn't have to compute expensive # nested string representations for obj in vars(_pyparsing).values(): try: if issubclass(obj, ParserElement): - obj.__str__ = object.__str__ - obj.__repr__ = object.__repr__ + obj.__str__ = functools.partial(fast_str, obj) + obj.__repr__ = functools.partial(fast_repr, obj) except TypeError: pass From a89f90c4bdc55dea5b71a71b8230810d10bd8875 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 16 Apr 2019 16:13:48 -0700 Subject: [PATCH 103/163] Fix tab completion Resolves #469. --- .travis.yml | 1 - coconut/_pyparsing.py | 1 + coconut/compiler/compiler.py | 6 ++++-- coconut/icoconut/root.py | 23 +++++++++++++++++++++-- coconut/root.py | 2 +- 5 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e8881ec9f..0aa6b1217 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ python: - '2.6' - '2.7' - pypy -- '3.3' - '3.5' - '3.6' - pypy3 diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index fa3c14f1d..a63e9dc1b 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -101,6 +101,7 @@ def fast_repr(cls): except TypeError: pass + if packrat_cache: ParserElement.enablePackrat(packrat_cache) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index d389426ab..d227d856e 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -470,11 +470,13 @@ def set_skips(self, skips): internal_assert(lambda: len(set(skips)) == len(skips), "duplicate line skip(s) in skips", skips) self.skips = skips - def adjust(self, ln): + def adjust(self, ln, skips=None): """Converts a parsing line number into an original line number.""" + if skips is None: + skips = self.skips adj_ln = ln need_unskipped = 0 - for i in self.skips: + for i in skips: if i <= ln: need_unskipped += 1 elif adj_ln + need_unskipped < i: diff --git a/coconut/icoconut/root.py b/coconut/icoconut/root.py index 521b7db2a..acda82637 100644 --- a/coconut/icoconut/root.py +++ b/coconut/icoconut/root.py @@ -22,7 +22,10 @@ import os import traceback -from coconut.exceptions import CoconutException +from coconut.exceptions import ( + CoconutException, + CoconutInternalException, +) from coconut.constants import ( py_syntax_version, mimetype, @@ -175,7 +178,7 @@ def user_expressions(self, expressions): class CoconutKernel(IPythonKernel, object): """Jupyter kernel for Coconut.""" shell_class = CoconutShell - use_experimental_completions = False + use_experimental_completions = True implementation = "icoconut" implementation_version = VERSION language = "coconut" @@ -202,3 +205,19 @@ class CoconutKernel(IPythonKernel, object): "url": documentation_url, }, ] + + def do_complete(self, code, cursor_pos): + # first try with Jedi completions + self.use_experimental_completions = True + try: + return super(CoconutKernel, self).do_complete(code, cursor_pos) + except Exception: + traceback.print_exc() + logger.warn_err(CoconutInternalException("experimental IPython completion failed, defaulting to shell completion")) + + # then if that fails default to shell completions + self.use_experimental_completions = False + try: + return super(CoconutKernel, self).do_complete(code, cursor_pos) + finally: + self.use_experimental_completions = True diff --git a/coconut/root.py b/coconut/root.py index e14662161..8d9b20cf0 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 29 +DEVELOP = 30 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From d9e61d8540808ef6cc97df60df557dea23601834 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 16 Apr 2019 18:18:22 -0700 Subject: [PATCH 104/163] Improve count --- DOCS.md | 4 +++- coconut/compiler/templates/header.py_template | 22 ++++++++++++------- coconut/constants.py | 2 ++ coconut/icoconut/root.py | 2 +- coconut/terminal.py | 4 ++-- tests/src/cocotest/agnostic/main.coco | 6 +++++ 6 files changed, 28 insertions(+), 12 deletions(-) diff --git a/DOCS.md b/DOCS.md index 5e9d85950..308df5538 100644 --- a/DOCS.md +++ b/DOCS.md @@ -2091,7 +2091,9 @@ collections.deque(map(print, map(lambda x: x**2, range(10))), maxlen=0) ### `count` -Coconut provides a modified version of `itertools.count` that supports `in`, normal slicing, optimized iterator slicing, the standard `count` and `index` sequence methods, `repr`, and `start`/`step` attributes as a built-in under the name `count`. Additionally, if the _step_ parameter is set to 0, `count` will infinitely repeat the _start_ parameter. +Coconut provides a modified version of `itertools.count` that supports `in`, normal slicing, optimized iterator slicing, the standard `count` and `index` sequence methods, `repr`, and `start`/`step` attributes as a built-in under the name `count`. + +Additionally, if the _step_ parameter is set to `None`, `count` will behave like `itertools.repeat` instead. ##### Python Docs diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index e07ba257a..4b2bd4b23 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -85,13 +85,15 @@ class reiterable{object}: __slots__ = ("iter",) def __init__(self, iterable): self.iter = iterable + def _get_new_iter(self): + self.iter, new_iter = _coconut_tee(self.iter) + return new_iter def __iter__(self): - self.iter, out = _coconut_tee(self.iter) - return _coconut.iter(out) + return _coconut.iter(self._get_new_iter()) def __getitem__(self, index): - return _coconut_igetitem(_coconut.iter(self), index) + return _coconut_igetitem(self._get_new_iter(), index) def __reversed__(self): - return _coconut_reversed(_coconut.iter(self)) + return _coconut_reversed(self._get_new_iter()) def __len__(self): return _coconut.len(self.iter) def __repr__(self): @@ -99,8 +101,7 @@ class reiterable{object}: def __reduce__(self): return (self.__class__, (self.iter,)) def __copy__(self): - self.iter, new_iter = _coconut_tee(self.iter) - return self.__class__(new_iter) + return self.__class__(self._get_new_iter()) def __fmap__(self, func): return _coconut_map(func, self) class scan{object}: @@ -311,10 +312,15 @@ class count{object}: return (elem - self.start) % self.step == 0 def __getitem__(self, index): if _coconut.isinstance(index, _coconut.slice) and (index.start is None or index.start >= 0) and (index.stop is None or index.stop >= 0): + new_start, new_step = self.start, self.step + if self.step and index.start is not None: + new_start += self.step * index.start + if self.step and index.step is not None: + new_step *= index.step if index.stop is None: - return self.__class__(self.start + (index.start if index.start is not None else 0), self.step * (index.step if index.step is not None else 1)) + return self.__class__(new_start, new_step) if self.step and _coconut.isinstance(self.start, _coconut.int) and _coconut.isinstance(self.step, _coconut.int): - return _coconut.range(self.start + self.step * (index.start if index.start is not None else 0), self.start + self.step * index.stop, self.step * (index.step if index.step is not None else 1)) + return _coconut.range(new_start, self.start + self.step * index.stop, new_step) return _coconut_map(self.__getitem__, _coconut.range(index.start if index.start is not None else 0, index.stop, index.step if index.step is not None else 1)) if index < 0: raise _coconut.IndexError("count indices must be positive") diff --git a/coconut/constants.py b/coconut/constants.py index dbd796a2c..2a7b4b4e8 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -518,6 +518,8 @@ def checksum(data): "io.StringIO": ("StringIO./StringIO", (2, 7)), "io.BytesIO": ("cStringIO./StringIO", (2, 7)), "importlib.reload": ("imp./reload", (3, 4)), + "itertools.filterfalse": ("itertools./ifilterfalse", (3,)), + "itertools.zip_longest": ("itertools./izip_longest", (3,)), # third-party backports "asyncio": ("trollius", (3, 4)), } diff --git a/coconut/icoconut/root.py b/coconut/icoconut/root.py index acda82637..7d8bcee4c 100644 --- a/coconut/icoconut/root.py +++ b/coconut/icoconut/root.py @@ -213,7 +213,7 @@ def do_complete(self, code, cursor_pos): return super(CoconutKernel, self).do_complete(code, cursor_pos) except Exception: traceback.print_exc() - logger.warn_err(CoconutInternalException("experimental IPython completion failed, defaulting to shell completion")) + logger.warn_err(CoconutInternalException("experimental IPython completion failed, defaulting to shell completion"), force=True) # then if that fails default to shell completions self.use_experimental_completions = False diff --git a/coconut/terminal.py b/coconut/terminal.py index 16c25ebcf..a0865b075 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -176,12 +176,12 @@ def warn(self, *args, **kwargs): """Creates and displays a warning.""" return self.warn_err(CoconutWarning(*args, **kwargs)) - def warn_err(self, warning): + def warn_err(self, warning, force=False): """Displays a warning.""" try: raise warning except Exception: - if not self.quiet: + if not self.quiet or force: self.display_exc() def display_exc(self): diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index c4fe1befa..51e59b887 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -530,6 +530,12 @@ def main_test(): assert str(err) == "(assert) got falsey value []" else: assert False + from itertools import filterfalse, zip_longest + assert reversed(reiterable(range(10)))[-1] == 0 + assert count("derp", None)[10] == "derp" + assert count("derp", None)[5:10] |> list == ["derp"] * 5 + assert count("derp", None)[5:] == count("derp", None) + assert count("derp", None)[:5] |> list == ["derp"] * 5 return True def tco_func() = tco_func() From 6f60eceb9a1f88b31da5ad78e70e89564c981473 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 16 Apr 2019 19:21:45 -0700 Subject: [PATCH 105/163] Fix failing tests --- coconut/requirements.py | 2 +- tests/src/extras.coco | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/coconut/requirements.py b/coconut/requirements.py index c72d14de1..d5751fec7 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -126,6 +126,7 @@ def everything_in(req_dict): "docs": unique_wrt(get_reqs("docs"), requirements), "tests": uniqueify_all( get_reqs("tests"), + get_reqs("purepython"), extras["jobs"] if not PYPY else [], extras["jupyter"] if IPY else [], extras["mypy"] if PY34 and not WINDOWS and not PYPY else [], @@ -135,7 +136,6 @@ def everything_in(req_dict): extras["dev"] = uniqueify_all( everything_in(extras), - get_reqs("purepython"), get_reqs("dev"), ) diff --git a/tests/src/extras.coco b/tests/src/extras.coco index 609229289..5e0289000 100644 --- a/tests/src/extras.coco +++ b/tests/src/extras.coco @@ -143,14 +143,12 @@ def test_extras(): assert "derp" in complete_result["matches"] assert complete_result["cursor_start"] == 0 assert complete_result["cursor_end"] == 1 - assert not complete_result["metadata"] keyword_complete_result = k.do_complete("ma", 1) assert keyword_complete_result["status"] == "ok" assert "match" in keyword_complete_result["matches"] assert "map" in keyword_complete_result["matches"] assert keyword_complete_result["cursor_start"] == 0 assert keyword_complete_result["cursor_end"] == 1 - assert not keyword_complete_result["metadata"] if not PYPY: import numpy as np assert np.all(fmap(-> _ + 1, np.arange(3)) == np.array([1, 2, 3])) From 54833606180b8079a3fdbfec2d1012847c6117f2 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 17 Apr 2019 14:15:36 -0700 Subject: [PATCH 106/163] Fix tests --- tests/main_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/main_test.py b/tests/main_test.py index 845db202b..f8d4ee150 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -265,7 +265,7 @@ def run(args=[], agnostic_target=None, use_run_arg=False, expect_retcode=0): if sys.version_info >= (3, 5): comp_35(args) comp_agnostic(agnostic_args, expect_retcode=expect_retcode) - comp_sys(args) + comp_sys(args, expect_retcode=expect_retcode) if use_run_arg: comp_runner(["--run"] + agnostic_args, expect_retcode=expect_retcode, assert_output=True) @@ -402,7 +402,7 @@ def test_jupyter_console(self): cmd = "coconut --jupyter console" print("\n>", cmd) p = pexpect.spawn(cmd) - p.expect("In", timeout=60) + p.expect("In", timeout=100) p.sendeof() p.sendline("y") p.expect("Shutting down kernel|shutting down|Jupyter error") From 8b4c1e95eebc8e6577353cf899d872513ce4c05c Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 17 Apr 2019 14:25:46 -0700 Subject: [PATCH 107/163] Remove mypy retcode checking --- tests/main_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/main_test.py b/tests/main_test.py index f8d4ee150..d93560f8b 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -420,10 +420,10 @@ def test_mypy_snip(self): call(["coconut", "-c", mypy_snip, "--mypy"], assert_output=mypy_snip_err, check_mypy=False, expect_retcode=1) def test_mypy(self): - run(["--mypy"] + mypy_args, expect_retcode=1) # fails due to tutorial mypy errors + run(["--mypy"] + mypy_args, expect_retcode=None) # fails due to tutorial mypy errors def test_mypy_sys(self): - run(["--mypy"] + mypy_args, agnostic_target="sys", expect_retcode=1) # fails due to tutorial mypy errors + run(["--mypy"] + mypy_args, agnostic_target="sys", expect_retcode=None) # fails due to tutorial mypy errors def test_target(self): run(agnostic_target=(2 if PY2 else 3)) From 86af17a59022426ab8e2cf1b32ad58d5bf3b698b Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 17 Apr 2019 16:17:27 -0700 Subject: [PATCH 108/163] Further improve retcode checking --- tests/main_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/main_test.py b/tests/main_test.py index d93560f8b..ef2858f2c 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -259,11 +259,11 @@ def run(args=[], agnostic_target=None, use_run_arg=False, expect_retcode=0): with using_dest(): if PY2: - comp_2(args) + comp_2(args, expect_retcode=expect_retcode) else: - comp_3(args) + comp_3(args, expect_retcode=expect_retcode) if sys.version_info >= (3, 5): - comp_35(args) + comp_35(args, expect_retcode=expect_retcode) comp_agnostic(agnostic_args, expect_retcode=expect_retcode) comp_sys(args, expect_retcode=expect_retcode) @@ -274,9 +274,9 @@ def run(args=[], agnostic_target=None, use_run_arg=False, expect_retcode=0): run_src() if use_run_arg: - comp_extras(["--run"] + agnostic_args, assert_output=True, check_errors=False, stderr_first=True) + comp_extras(["--run"] + agnostic_args, assert_output=True, check_errors=False, stderr_first=True, expect_retcode=expect_retcode) else: - comp_extras(agnostic_args) + comp_extras(agnostic_args, expect_retcode=expect_retcode) run_extras() From 3255fc04cc4494a53df836f63940d518cf51b84c Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 23 Apr 2019 18:28:00 -0700 Subject: [PATCH 109/163] Fix asyncio mypy error --- tests/src/extras.coco | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/extras.coco b/tests/src/extras.coco index 5e0289000..dce811ee4 100644 --- a/tests/src/extras.coco +++ b/tests/src/extras.coco @@ -26,7 +26,7 @@ from coconut.convenience import ( if IPY and not WINDOWS: if PY35: - import asyncio + import asyncio # type: ignore from coconut.icoconut import CoconutKernel # type: ignore else: CoconutKernel = None # type: ignore From 9d1a495e3f0882c0244827f8e58205385409f8d4 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 8 May 2019 13:29:52 -0700 Subject: [PATCH 110/163] Bump mypy version --- coconut/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/constants.py b/coconut/constants.py index 2a7b4b4e8..a83eae38e 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -177,7 +177,7 @@ def checksum(data): "recommonmark": (0, 5), "psutil": (5,), "jupyter": (1, 0), - "mypy": (0, 700), + "mypy": (0, 701), "futures": (3, 2), "backports.functools-lru-cache": (1, 5), "argparse": (1, 4), From b42a019a72e866e48edad8edd2585f5bff8973fb Mon Sep 17 00:00:00 2001 From: Mikhail Burshteyn Date: Sun, 19 May 2019 18:23:25 +0300 Subject: [PATCH 111/163] Parse f-string expressions as Coconut This allows writing code like `f"{[] |> len}"`. Fixes #504. --- CONTRIBUTING.md | 3 + coconut/compiler/compiler.py | 128 +++++++++++--------- coconut/compiler/util.py | 14 +++ tests/main_test.py | 8 ++ tests/src/cocotest/agnostic/main.coco | 4 + tests/src/cocotest/target_36/py36_test.coco | 4 + 6 files changed, 106 insertions(+), 55 deletions(-) create mode 100644 tests/src/cocotest/target_36/py36_test.coco diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4f501f3a0..a8c6b7929 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -152,6 +152,9 @@ After you've tested your changes locally, you'll want to add more permanent test - python35 - `py35_test.coco` + Tests to be run only on Python 3.5 with `--target 3.5`. + - python36 + - `py36_test.coco` + + Tests to be run only on Python 3.6 with `--target 3.6`. ## Release Process diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index d227d856e..92a7c1b26 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -117,6 +117,7 @@ compile_regex, keyword, append_it, + interleaved_join, ) from coconut.compiler.header import ( minify, @@ -368,6 +369,13 @@ def __reduce__(self): """Return pickling information.""" return (Compiler, (self.target, self.strict, self.minify, self.line_numbers, self.keep_lines, self.no_tco)) + def __copy__(self): + """Create a copy of this object.""" + cls, args = self.__reduce__() + return cls(*args) + + copy = __copy__ + def genhash(self, code, package_level=-1): """Generate a hash from code.""" reduce_args = self.__reduce__()[1] @@ -1992,66 +2000,76 @@ def f_string_handle(self, original, loc, tokens): """Handle Python 3.6 format strings.""" internal_assert(len(tokens) == 1, "invalid format string tokens", tokens) string = tokens[0] - if self.target_info >= (3, 6): - return "f" + string - else: - # strip raw r - raw = string.startswith("r") - if raw: - string = string[1:] - - # strip wrappers - internal_assert(string.startswith(strwrapper) and string.endswith(unwrapper)) - string = string[1:-1] - - # get text - old_text, strchar = self.get_ref("str", string) - - # separate expressions - new_text = "" - exprs = [] - saw_brace = False - in_expr = False - expr_level = 0 - for c in old_text: - if saw_brace: - saw_brace = False - if c == "{": - new_text += c - elif c == "}": - raise self.make_err(CoconutSyntaxError, "empty expressing in format string", original, loc) - else: - in_expr = True - expr_level = paren_change(c) - exprs.append(c) - elif in_expr: - if expr_level < 0: - expr_level += paren_change(c) - exprs[-1] += c - elif expr_level > 0: - raise self.make_err(CoconutSyntaxError, "imbalanced parentheses in format string expression", original, loc) - elif c in "!:}": # these characters end the expr - in_expr = False - name = format_var + "_" + str(len(exprs) - 1) - new_text += name + c - else: - exprs[-1] += c - elif c == "{": - saw_brace = True - new_text += c - else: - new_text += c - # handle dangling detections + # strip raw r + raw = string.startswith("r") + if raw: + string = string[1:] + + # strip wrappers + internal_assert(string.startswith(strwrapper) and string.endswith(unwrapper)) + string = string[1:-1] + + # get text + old_text, strchar = self.get_ref("str", string) + + # separate expressions + string_parts = [""] + exprs = [] + saw_brace = False + in_expr = False + expr_level = 0 + for c in old_text: if saw_brace: - raise self.make_err(CoconutSyntaxError, "format string ends with unescaped brace (escape by doubling to '{{')", original, loc) - if in_expr: - raise self.make_err(CoconutSyntaxError, "imbalanced braces in format string (escape braces by doubling to '{{' and '}}')", original, loc) + saw_brace = False + if c == "{": + string_parts[-1] += c + elif c == "}": + raise self.make_err(CoconutSyntaxError, "empty expressing in format string", original, loc) + else: + in_expr = True + expr_level = paren_change(c) + exprs.append(c) + elif in_expr: + if expr_level < 0: + expr_level += paren_change(c) + exprs[-1] += c + elif expr_level > 0: + raise self.make_err(CoconutSyntaxError, "imbalanced parentheses in format string expression", original, loc) + elif c in "!:}": # these characters end the expr + in_expr = False + string_parts.append(c) + else: + exprs[-1] += c + elif c == "{": + saw_brace = True + string_parts[-1] += c + else: + string_parts[-1] += c + + # handle dangling detections + if saw_brace: + raise self.make_err(CoconutSyntaxError, "format string ends with unescaped brace (escape by doubling to '{{')", original, loc) + if in_expr: + raise self.make_err(CoconutSyntaxError, "imbalanced braces in format string (escape braces by doubling to '{{' and '}}')", original, loc) + + expr_compiler = self.copy() + compiled_exprs = [ + expr_compiler.parse_eval(expr) + for expr in exprs + ] + self.bind() # hack to reset `Forward`s in grammar + + if self.target_info >= (3, 6): + return "f" + ("r" if raw else "") + strchar + interleaved_join(string_parts, compiled_exprs) + strchar + else: + names = [format_var + "_" + str(i) for i in range(len(compiled_exprs))] + new_text = interleaved_join(string_parts, names) # generate format call return ("r" if raw else "") + strchar + new_text + strchar + ".format(" + ", ".join( - format_var + "_" + str(i) + "=(" + expr + ")" - for i, expr in enumerate(exprs) + name + "=(" + expr + ")" + for name, expr in zip(names, compiled_exprs) ) + ")" # end: COMPILER HANDLERS diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 7c9f84b8c..cc4ae38b5 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -587,3 +587,17 @@ def disable_outside(item, *elems): """ for wrapped in disable_inside(item, *elems, **{"_invert": True}): yield wrapped + + +def interleaved_join(outer_list, inner_list): + """Interleaves two lists of strings and joins the result. + + Example: interleaved_join(['1', '3'], ['2']) == '123' + The first list must be 1 longer than the second list. + """ + internal_assert(len(outer_list) == len(inner_list) + 1, "invalid list lengths to interleaved_join") + interleaved = [] + for xx in zip(outer_list, inner_list): + interleaved.extend(xx) + interleaved.append(outer_list[-1]) + return "".join(interleaved) diff --git a/tests/main_test.py b/tests/main_test.py index ef2858f2c..55c3a2b83 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -234,6 +234,11 @@ def comp_35(args=[], **kwargs): comp(path="cocotest", folder="target_35", args=["--target", "35"] + args, **kwargs) +def comp_36(args=[], **kwargs): + """Compiles target_35.""" + comp(path="cocotest", folder="target_36", args=["--target", "36"] + args, **kwargs) + + def comp_sys(args=[], **kwargs): """Compiles target_sys.""" comp(path="cocotest", folder="target_sys", args=["--target", "sys"] + args, **kwargs) @@ -264,6 +269,8 @@ def run(args=[], agnostic_target=None, use_run_arg=False, expect_retcode=0): comp_3(args, expect_retcode=expect_retcode) if sys.version_info >= (3, 5): comp_35(args, expect_retcode=expect_retcode) + if sys.version_info >= (3, 6): + comp_36(args, expect_retcode=expect_retcode) comp_agnostic(agnostic_args, expect_retcode=expect_retcode) comp_sys(args, expect_retcode=expect_retcode) @@ -328,6 +335,7 @@ def comp_all(args=[], **kwargs): comp_2(args, **kwargs) comp_3(args, **kwargs) comp_35(args, **kwargs) + comp_36(args, **kwargs) comp_agnostic(args, **kwargs) comp_sys(args, **kwargs) comp_runner(args, **kwargs) diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index 51e59b887..dd444b923 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -509,6 +509,7 @@ def main_test(): assert f"{{" == "{" assert f"}}" == "}" assert f"{1, 2}" == "(1, 2)" + assert f"{[] |> len}" == "0" match {"a": {"b": x }} or {"a": {"b": {"c": x}}} = {"a": {"b": {"c": "x"}}} assert x == {"c": "x"} assert py_repr("x") == ("u'x'" if sys.version_info < (3,) else "'x'") @@ -582,6 +583,9 @@ def main(*args): if sys.version_info >= (3, 5): from .py35_test import py35_test assert py35_test() + if sys.version_info >= (3, 6): + from .py36_test import py36_test + assert py36_test() print(".", end="") from .target_sys_test import target_sys_test diff --git a/tests/src/cocotest/target_36/py36_test.coco b/tests/src/cocotest/target_36/py36_test.coco new file mode 100644 index 000000000..1d483808f --- /dev/null +++ b/tests/src/cocotest/target_36/py36_test.coco @@ -0,0 +1,4 @@ +def py36_test(): + """Performs Python-3.6-specific tests.""" + assert f"{[] |> len}" == "0" + return True From 2e3dedcd38a3c154b16f6f6764e88353978b9e4d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 11 Jun 2019 17:46:21 -0700 Subject: [PATCH 112/163] Avoid copying compiler Provides a significant speedup. --- coconut/compiler/compiler.py | 55 +++++++++++++++++++++++++++--------- coconut/compiler/util.py | 2 +- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 92a7c1b26..704ead11f 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -370,7 +370,7 @@ def __reduce__(self): return (Compiler, (self.target, self.strict, self.minify, self.line_numbers, self.keep_lines, self.no_tco)) def __copy__(self): - """Create a copy of this object.""" + """Create a new, blank copy of the compiler.""" cls, args = self.__reduce__() return cls(*args) @@ -401,16 +401,31 @@ def reset(self): self.indchar = None self.comments = {} self.refs = [] - self.set_skips([]) + self.skips = [] self.docstring = "" self.ichain_count = 0 self.tre_store_count = 0 self.case_check_count = 0 self.stmt_lambdas = [] - if self.strict: - self.unused_imports = set() + self.unused_imports = set() + self.original_lines = [] self.bind() + @contextmanager + def inner_environment(self): + """Set up compiler to evaluate inner expressions.""" + comments, self.comments = self.comments, {} + skips, self.skips = self.skips, [] + docstring, self.docstring = self.docstring, "" + original_lines, self.original_lines = self.original_lines, [] + try: + yield + finally: + self.comments = comments + self.skips = skips + self.docstring = docstring + self.original_lines = original_lines + def bind(self): """Binds reference objects to the proper parse actions.""" self.endline <<= attach(self.endline_ref, self.endline_handle) @@ -637,6 +652,15 @@ def make_parse_err(self, err, reformat=True, include_ln=True): err_lineno = self.adjust(err_lineno) return CoconutParseError(None, err_line, err_index, err_lineno) + def inner_parse_eval(self, inputstring, parser=None, preargs={"strip": True}, postargs={"header": "none", "initial": "none", "final_endline": False}): + """Parse eval code in an inner environment.""" + if parser is None: + parser = self.eval_parser + with self.inner_environment(): + pre_procd = self.pre(inputstring, **preargs) + parsed = parse(parser, pre_procd) + return self.post(parsed, **postargs) + def parse(self, inputstring, parser, preargs, postargs): """Use the parser to parse the inputstring with appropriate setup and teardown.""" self.reset() @@ -2025,7 +2049,7 @@ def f_string_handle(self, original, loc, tokens): if c == "{": string_parts[-1] += c elif c == "}": - raise self.make_err(CoconutSyntaxError, "empty expressing in format string", original, loc) + raise self.make_err(CoconutSyntaxError, "empty expression in format string", original, loc) else: in_expr = True expr_level = paren_change(c) @@ -2053,21 +2077,26 @@ def f_string_handle(self, original, loc, tokens): if in_expr: raise self.make_err(CoconutSyntaxError, "imbalanced braces in format string (escape braces by doubling to '{{' and '}}')", original, loc) - expr_compiler = self.copy() - compiled_exprs = [ - expr_compiler.parse_eval(expr) - for expr in exprs - ] - self.bind() # hack to reset `Forward`s in grammar + # compile Coconut expressions + compiled_exprs = [] + for co_expr in exprs: + py_expr = self.inner_parse_eval(co_expr) + if "\n" in py_expr: + print(py_expr) + raise self.make_err(CoconutSyntaxError, "invalid expression in format string: " + co_expr, original, loc) + compiled_exprs.append(py_expr) + # reconstitute string if self.target_info >= (3, 6): - return "f" + ("r" if raw else "") + strchar + interleaved_join(string_parts, compiled_exprs) + strchar + new_text = interleaved_join(string_parts, compiled_exprs) + + return "f" + ("r" if raw else "") + self.wrap_str(new_text, strchar) else: names = [format_var + "_" + str(i) for i in range(len(compiled_exprs))] new_text = interleaved_join(string_parts, names) # generate format call - return ("r" if raw else "") + strchar + new_text + strchar + ".format(" + ", ".join( + return ("r" if raw else "") + self.wrap_str(new_text, strchar) + ".format(" + ", ".join( name + "=(" + expr + ")" for name, expr in zip(names, compiled_exprs) ) + ")" diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index cc4ae38b5..d09142000 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -595,7 +595,7 @@ def interleaved_join(outer_list, inner_list): Example: interleaved_join(['1', '3'], ['2']) == '123' The first list must be 1 longer than the second list. """ - internal_assert(len(outer_list) == len(inner_list) + 1, "invalid list lengths to interleaved_join") + internal_assert(len(outer_list) == len(inner_list) + 1, "invalid list lengths to interleaved_join", (outer_list, inner_list)) interleaved = [] for xx in zip(outer_list, inner_list): interleaved.extend(xx) From 333722fb213097fdc3a24bbf397b38326cfba19b Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 12 Jun 2019 18:42:31 -0700 Subject: [PATCH 113/163] Attempt to fix addpattern, tco pickling Remaining problem: inner func not pickleable bc var name is rebound to outer func. --- coconut/compiler/header.py | 33 +++++++----- coconut/compiler/templates/header.py_template | 52 ++++++++++++------- tests/src/cocotest/agnostic/main.coco | 3 +- tests/src/cocotest/agnostic/suite.coco | 5 +- 4 files changed, 59 insertions(+), 34 deletions(-) diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index e80b9f587..80438966a 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -175,8 +175,11 @@ class you_need_to_install_trollius: pass with ThreadPoolExecutor(cpu_count() * 5)''' if target_info < (3, 5) else '''with ThreadPoolExecutor()''' ), - tco_decorator="@_coconut_tco\n" + " " * 8 if not no_tco else "", - tail_call_func_args_kwargs="func(*args, **kwargs)" if no_tco else "_coconut_tail_call(func, *args, **kwargs)", + tail_call_last_pattern="self.patterns[-1](*args, **kwargs)" if no_tco else "_coconut_tail_call(self.patterns[-1], *args, **kwargs)", + tco_partial_base_pattern_func=( + "_coconut.functools.partial(_coconut_base_pattern_func, base_func)" if no_tco + else "_coconut_back_compose(_coconut_tco, _coconut.functools.partial(_coconut_base_pattern_func, base_func))" + ), comma_tco=", _coconut_tail_call, _coconut_tco" if not no_tco else "", def_prepattern=( r'''def prepattern(base_func): @@ -204,28 +207,30 @@ def NamedTuple(name, fields): format_dict["underscore_imports"] = "_coconut, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_back_pipe, _coconut_star_pipe, _coconut_back_star_pipe, _coconut_dubstar_pipe, _coconut_back_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_addpattern, _coconut_sentinel, _coconut_assert".format(**format_dict) - # ._coconut_tco_func is used in main.coco, so don't remove it + # .func is used in main.coco, so don't remove it # here without replacing its usage there format_dict["def_tco"] = "" if no_tco else '''class _coconut_tail_call{object}: __slots__ = ("func", "args", "kwargs") def __init__(self, func, *args, **kwargs): self.func, self.args, self.kwargs = func, args, kwargs -_coconut_tco_func_dict = {empty_dict} -def _coconut_tco(func): - @_coconut.functools.wraps(func) - def tail_call_optimized_func(*args, **kwargs): - call_func = func +class _coconut_tco{object}: + __slots__ = ("__doc__", "func") + def __init__(self, func): + self.__doc__ = func.__doc__ + self.func = func + def __call__(self, *args, **kwargs): + call_func = self.func while True: - wkref = _coconut_tco_func_dict.get(_coconut.id(call_func)) - if wkref is not None and wkref() is call_func: - call_func = call_func._coconut_tco_func + if _coconut.isinstance(call_func, _coconut_tco): + call_func = call_func.func result = call_func(*args, **kwargs) # pass --no-tco to clean up your traceback if not isinstance(result, _coconut_tail_call): return result call_func, args, kwargs = result.func, result.args, result.kwargs - tail_call_optimized_func._coconut_tco_func = func - _coconut_tco_func_dict[_coconut.id(tail_call_optimized_func)] = _coconut.weakref.ref(tail_call_optimized_func) - return tail_call_optimized_func + def __repr__(self): + return repr(self.func) + def __reduce__(self): + return (self.__class__, (self.func,)) '''.format(**format_dict) return format_dict, target_startswith, target_info diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 4b2bd4b23..18c0fd2d9 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -4,7 +4,7 @@ class _coconut{object}:{comment.everything_here_must_be_copied_to_stub_file} {import_OrderedDict} {import_collections_abc} {import_typing_NamedTuple} - Ellipsis, Exception, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, classmethod, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, map, min, max, next, object, property, range, reversed, set, slice, str, sum, super, tuple, zip, repr{comma_bytearray} = Ellipsis, Exception, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, classmethod, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, map, min, max, next, object, property, range, reversed, set, slice, str, sum, super, tuple, zip, {static_repr}{comma_bytearray} + Ellipsis, Exception, AttributeError, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, classmethod, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, map, min, max, next, object, property, range, reversed, set, slice, str, sum, super, tuple, type, zip, repr{comma_bytearray} = Ellipsis, Exception, AttributeError, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, classmethod, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, map, min, max, next, object, property, range, reversed, set, slice, str, sum, super, tuple, type, zip, {static_repr}{comma_bytearray} _coconut_sentinel = _coconut.object() class MatchError(Exception): """Pattern-matching error. Has attributes .pattern and .value.""" @@ -30,7 +30,7 @@ class _coconut_base_compose{object}: self.func = func self.funcstars = [] for f, stars in funcstars: - if isinstance(f, _coconut_base_compose): + if _coconut.isinstance(f, _coconut_base_compose): self.funcstars.append((f.func, stars)) self.funcstars += f.funcstars else: @@ -45,7 +45,7 @@ class _coconut_base_compose{object}: elif stars == 2: arg = f(**arg) else: - raise ValueError() + raise _coconut.ValueError("invalid arguments to " + _coconut.repr(self)) return arg def __repr__(self): return _coconut.repr(self.func) + " " + " ".join(("..*> " if star == 1 else "..**>" if star == 2 else "..> ") + _coconut.repr(f) for f, star in self.funcstars) @@ -427,7 +427,7 @@ class _coconut_FunctionMatchErrorContext(object): def __enter__(self): try: self.threadlocal_var.contexts.append(self) - except AttributeError: + except _coconut.AttributeError: self.threadlocal_var.contexts = [self] def __exit__(self, type, value, traceback): self.threadlocal_var.contexts.pop() @@ -435,27 +435,43 @@ class _coconut_FunctionMatchErrorContext(object): def get(cls): try: ctx = cls.threadlocal_var.contexts[-1] - except (AttributeError, IndexError): - return MatchError + except (_coconut.AttributeError, _coconut.IndexError): + return _coconut_MatchError if not ctx.taken: ctx.taken = True return ctx.exc_class - return MatchError + return _coconut_MatchError _coconut_get_function_match_error = _coconut_FunctionMatchErrorContext.get +class _coconut_base_pattern_func{object}: + __slots__ = ("FunctionMatchError", "__doc__", "patterns") + def __init__(self, *funcs): + self.FunctionMatchError = _coconut.type(_coconut_str("MatchError"), (_coconut_MatchError,), {{}}) + self.__doc__ = None + self.patterns = [] + for func in funcs: + self.add(func) + def add(self, func): + self.__doc__ = func.__doc__ or self.__doc__ + if _coconut.isinstance(func, _coconut_base_pattern_func): + self.patterns += func.patterns + else: + self.patterns.append(func) + def __call__(self, *args, **kwargs): + for func in self.patterns[:-1]: + try: + with _coconut_FunctionMatchErrorContext(self.FunctionMatchError): + return func(*args, **kwargs) + except self.FunctionMatchError: + pass + return {tail_call_last_pattern} + def __repr__(self): + return "addpattern(" + _coconut.repr(self.patterns[0]) + ")(*" + _coconut.repr(self.patterns[1:]) + ")" + def __reduce__(self): + return (self.__class__, self.patterns) def addpattern(base_func): """Decorator to add a new case to a pattern-matching function, where the new case is checked last.""" - def pattern_adder(func): - FunctionMatchError = type(_coconut_str("MatchError"), (MatchError,), {{}}) - {tco_decorator}@_coconut.functools.wraps(func) - def add_pattern_func(*args, **kwargs): - try: - with _coconut_FunctionMatchErrorContext(FunctionMatchError): - return base_func(*args, **kwargs) - except FunctionMatchError: - return {tail_call_func_args_kwargs} - return add_pattern_func - return pattern_adder + return {tco_partial_base_pattern_func} _coconut_addpattern = addpattern {def_prepattern}class _coconut_partial{object}: __slots__ = ("func", "_argdict", "_arglen", "_stargs", "keywords") diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index dd444b923..d00d805ef 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -570,7 +570,8 @@ def main(*args): assert suite_test() print(".", end="") # ..... - if hasattr(tco_func, "_coconut_tco_func"): + if "_coconut_tco" in globals() or "_coconut_tco" in locals(): + assert hasattr(tco_func, "func") assert tco_test() print(".", end="") # ...... diff --git a/tests/src/cocotest/agnostic/suite.coco b/tests/src/cocotest/agnostic/suite.coco index dbfc024c9..149f2f73b 100644 --- a/tests/src/cocotest/agnostic/suite.coco +++ b/tests/src/cocotest/agnostic/suite.coco @@ -41,9 +41,12 @@ def suite_test(): assert zipsum([1,2,3], [10,20,30]) |> list == [11,22,33] assert clean(" ab cd ef ") == "ab cd ef" == " ab cd ef " |> clean assert add2 <| 2 <| 3 == 5 - for qsort in [qsort1, qsort2, qsort3, qsort4, qsort5, qsort6, qsort7]: + qsorts = [qsort1, qsort2, qsort3, qsort4, qsort5, qsort6, qsort7, qsort8] + for qsort in qsorts: to_sort = rand_list(10) assert to_sort |> qsort |> tuple == to_sort |> sorted |> tuple, qsort + to_sort = rand_list(10) + assert parallel_map(tuple <.. (|>)$(to_sort), qsorts) |> list == [to_sort |> sorted |> tuple] * len(qsorts) assert repeat(3)$[2] == 3 == repeat_(3)$[2] assert sum_(repeat(1)$[:5]) == 5 == sum_(repeat_(1)$[:5]) assert (sum_(takewhile((x)-> x<5, N())) From 17eca17eda90f2289d462cfe3d42c39319c7946d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 12 Jun 2019 20:48:58 -0700 Subject: [PATCH 114/163] Fix addpattern, tco pickling --- coconut/compiler/header.py | 31 ++++++++++--------- coconut/compiler/templates/header.py_template | 7 +++-- tests/src/cocotest/agnostic/main.coco | 2 +- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 80438966a..358570e2d 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -205,32 +205,33 @@ def NamedTuple(name, fields): return _coconut.collections.namedtuple(name, [x for x, t in fields])'''.format(**format_dict), ) - format_dict["underscore_imports"] = "_coconut, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_back_pipe, _coconut_star_pipe, _coconut_back_star_pipe, _coconut_dubstar_pipe, _coconut_back_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_addpattern, _coconut_sentinel, _coconut_assert".format(**format_dict) + format_dict["underscore_imports"] = "_coconut, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_back_pipe, _coconut_star_pipe, _coconut_back_star_pipe, _coconut_dubstar_pipe, _coconut_back_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert".format(**format_dict) - # .func is used in main.coco, so don't remove it + # ._coconut_tco_func is used in main.coco, so don't remove it # here without replacing its usage there format_dict["def_tco"] = "" if no_tco else '''class _coconut_tail_call{object}: __slots__ = ("func", "args", "kwargs") def __init__(self, func, *args, **kwargs): self.func, self.args, self.kwargs = func, args, kwargs -class _coconut_tco{object}: - __slots__ = ("__doc__", "func") - def __init__(self, func): - self.__doc__ = func.__doc__ - self.func = func - def __call__(self, *args, **kwargs): - call_func = self.func +_coconut_tco_func_dict = {empty_dict} +def _coconut_tco(func): + @_coconut.functools.wraps(func) + def tail_call_optimized_func(*args, **kwargs): + call_func = func while True: - if _coconut.isinstance(call_func, _coconut_tco): - call_func = call_func.func + wkref = _coconut_tco_func_dict.get(_coconut.id(call_func)) + if wkref is not None and wkref() is call_func: + call_func = call_func._coconut_tco_func result = call_func(*args, **kwargs) # pass --no-tco to clean up your traceback if not isinstance(result, _coconut_tail_call): return result call_func, args, kwargs = result.func, result.args, result.kwargs - def __repr__(self): - return repr(self.func) - def __reduce__(self): - return (self.__class__, (self.func,)) + tail_call_optimized_func._coconut_tco_func = func + tail_call_optimized_func.__module__ = _coconut.getattr(func, "__module__", None) + tail_call_optimized_func.__name__ = _coconut.getattr(func, "__name__", "") + tail_call_optimized_func.__qualname__ = _coconut.getattr(func, "__qualname__", tail_call_optimized_func.__name__) + _coconut_tco_func_dict[_coconut.id(tail_call_optimized_func)] = _coconut.weakref.ref(tail_call_optimized_func) + return tail_call_optimized_func '''.format(**format_dict) return format_dict, target_startswith, target_info diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 18c0fd2d9..37b1ae0e2 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -443,15 +443,18 @@ class _coconut_FunctionMatchErrorContext(object): return _coconut_MatchError _coconut_get_function_match_error = _coconut_FunctionMatchErrorContext.get class _coconut_base_pattern_func{object}: - __slots__ = ("FunctionMatchError", "__doc__", "patterns") def __init__(self, *funcs): self.FunctionMatchError = _coconut.type(_coconut_str("MatchError"), (_coconut_MatchError,), {{}}) self.__doc__ = None + self.__name__ = "" + self.__module__ = None self.patterns = [] for func in funcs: self.add(func) def add(self, func): - self.__doc__ = func.__doc__ or self.__doc__ + self.__doc__ = _coconut.getattr(func, "__doc__", None) or self.__doc__ + self.__name__ = _coconut.getattr(func, "__name__", None) or self.__name__ + self.__module__ = _coconut.getattr(func, "__module__", None) or self.__module__ if _coconut.isinstance(func, _coconut_base_pattern_func): self.patterns += func.patterns else: diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index d00d805ef..0842677df 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -571,7 +571,7 @@ def main(*args): print(".", end="") # ..... if "_coconut_tco" in globals() or "_coconut_tco" in locals(): - assert hasattr(tco_func, "func") + assert hasattr(tco_func, "_coconut_tco_func") assert tco_test() print(".", end="") # ...... From 894c1f58eea9584b59ed8dd7cd31775f23a55682 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 12 Jun 2019 22:11:39 -0700 Subject: [PATCH 115/163] Make recursive_iterator pickleable Resolves #502. --- coconut/compiler/templates/header.py_template | 27 +++++++++++-------- tests/src/cocotest/agnostic/suite.coco | 1 + tests/src/cocotest/agnostic/util.coco | 5 ++-- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 37b1ae0e2..ff4588e50 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -386,12 +386,14 @@ class groupsof{object}: return self.__class__(self.group_size, _coconut.copy.copy(self.iter)) def __fmap__(self, func): return _coconut_map(func, self) -def recursive_iterator(func): +class recursive_iterator{object}: """Decorator that optimizes a function for iterator recursion.""" - tee_store = {empty_dict} - backup_tee_store = [] - @_coconut.functools.wraps(func) - def recursive_iterator_func(*args, **kwargs): + __slots__ = ("func", "tee_store", "backup_tee_store") + def __init__(self, func): + self.func = func + self.tee_store = {empty_dict} + self.backup_tee_store = [] + def __call__(self, *args, **kwargs): key = (args, _coconut.frozenset(kwargs)) use_backup = False try: @@ -402,22 +404,25 @@ def recursive_iterator(func): except _coconut.Exception: use_backup = True if use_backup: - for i, (k, v) in _coconut.enumerate(backup_tee_store): + for i, (k, v) in _coconut.enumerate(self.backup_tee_store): if k == key: to_tee, store_pos = v, i break else: # no break - to_tee = func(*args, **kwargs) + to_tee = self.func(*args, **kwargs) store_pos = None to_store, to_return = _coconut_tee(to_tee) if store_pos is None: - backup_tee_store.append([key, to_store]) + self.backup_tee_store.append([key, to_store]) else: - backup_tee_store[store_pos][1] = to_store + self.backup_tee_store[store_pos][1] = to_store else: - tee_store[key], to_return = _coconut_tee(tee_store.get(key) or func(*args, **kwargs)) + self.tee_store[key], to_return = _coconut_tee(self.tee_store.get(key) or self.func(*args, **kwargs)) return to_return - return recursive_iterator_func + def __repr__(self): + return "@recursive_iterator(" + _coconut.repr(self.func) + ")" + def __reduce__(self): + return (self.__class__, (self.func,)) class _coconut_FunctionMatchErrorContext(object): __slots__ = ('exc_class', 'taken') threadlocal_var = _coconut.threading.local() diff --git a/tests/src/cocotest/agnostic/suite.coco b/tests/src/cocotest/agnostic/suite.coco index 149f2f73b..ed0a016df 100644 --- a/tests/src/cocotest/agnostic/suite.coco +++ b/tests/src/cocotest/agnostic/suite.coco @@ -244,6 +244,7 @@ def suite_test(): assert fibs()$[1:4] |> tuple == (1, 2, 3) assert fibs() |> takewhile$((i) -> i < 4000000 ) |> filter$((i) -> i % 2 == 0 ) |> sum == 4613732 assert loop([1,2])$[:4] |> list == [1, 2] * 2 + assert parallel_map(list .. .$[:2] .. loop, ([1], [2]))$[:2] |> tuple == ([1, 1], [2, 2]) assert (def -> mod)()(5, 3) == 2 assert sieve((2, 3, 4, 5)) |> list == [2, 3, 5] assert 11 == double_plus_one(5) diff --git a/tests/src/cocotest/agnostic/util.coco b/tests/src/cocotest/agnostic/util.coco index 0f8dcde1c..f0e945361 100644 --- a/tests/src/cocotest/agnostic/util.coco +++ b/tests/src/cocotest/agnostic/util.coco @@ -652,8 +652,9 @@ addpattern def `pattern_abs_` (x) = x # type: ignore @recursive_iterator def fibs() = (1, 1) :: map((+), fibs(), fibs()$[1:]) -@recursive_iterator -def loop(it) = it :: loop(it) +# use separate name for base func for pickle +def _loop(it) = it :: loop(it) +loop = recursive_iterator(_loop) # Sieve Example From 945a2f83d40ecb90e9a820da5194bcb1597a492a Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 13 Jun 2019 13:08:23 -0700 Subject: [PATCH 116/163] Bump develop version --- coconut/root.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/root.py b/coconut/root.py index 8d9b20cf0..9ab4dea7d 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 30 +DEVELOP = 31 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 79f232c367f4817980cd9c570172d3f2a70fd4c7 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 13 Jun 2019 18:01:39 -0700 Subject: [PATCH 117/163] Add Python 3.8 pos-only args Resolves #503. --- DOCS.md | 5 ++-- coconut/compiler/compiler.py | 33 ++++++++++++++++++++++----- coconut/compiler/grammar.py | 16 +++++++++++-- coconut/compiler/matching.py | 17 ++++++++------ coconut/root.py | 2 +- tests/src/cocotest/agnostic/main.coco | 9 ++++++++ tests/src/extras.coco | 1 + 7 files changed, 65 insertions(+), 18 deletions(-) diff --git a/DOCS.md b/DOCS.md index 308df5538..2d0c86f97 100644 --- a/DOCS.md +++ b/DOCS.md @@ -206,8 +206,9 @@ Finally, while Coconut will try to compile Python-3-specific syntax to its unive - destructuring assignment with `*`s (use pattern-matching instead), - tuples and lists with `*` unpacking or dicts with `**` unpacking (requires `--target 3.5`), - `@` as matrix multiplication (requires `--target 3.5`), -- `async` and `await` statements (requires `--target 3.5`), and -- `:=` assignment expressions (requires `--target 3.8`). +- `async` and `await` statements (requires `--target 3.5`), +- `:=` assignment expressions (requires `--target 3.8`), and +- positional-only function arguments (use pattern-matching function definition instead) (requires `--target 3.8`). ### Allowable Targets diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 704ead11f..b1fa0673d 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -257,7 +257,12 @@ def imported_names(imports): def split_args_list(tokens, loc): """Splits function definition arguments.""" - req_args, def_args, star_arg, kwd_args, dubstar_arg = [], [], None, [], None + pos_only_args = [] + req_args = [] + def_args = [] + star_arg = None + kwd_args = [] + dubstar_arg = None pos = 0 for arg in tokens: if len(arg) == 1: @@ -266,6 +271,16 @@ def split_args_list(tokens, loc): if pos >= 3: raise CoconutDeferredSyntaxError("star separator at invalid position in function definition", loc) pos = 3 + elif arg[0] == "/": + # slash sep (pos = 0) + if pos > 0: + raise CoconutDeferredSyntaxError("slash separator at invalid position in function definition", loc) + if pos_only_args: + raise CoconutDeferredSyntaxError("only one slash separator allowed in function definition", loc) + if not req_args: + raise CoconutDeferredSyntaxError("slash separator must come after arguments to mark as positional-only") + pos_only_args = req_args + req_args = [] else: # pos arg (pos = 0) if pos > 0: @@ -297,7 +312,7 @@ def split_args_list(tokens, loc): raise CoconutDeferredSyntaxError("invalid default argument in function definition", loc) else: raise CoconutInternalException("invalid function definition argument", arg) - return req_args, def_args, star_arg, kwd_args, dubstar_arg + return pos_only_args, req_args, def_args, star_arg, kwd_args, dubstar_arg def match_case_tokens(loc, tokens, check_var, top): @@ -478,6 +493,8 @@ def bind(self): self.dubstar_expr <<= attach(self.dubstar_expr_ref, self.star_expr_check) self.star_sep_arg <<= attach(self.star_sep_arg_ref, self.star_sep_check) self.star_sep_vararg <<= attach(self.star_sep_vararg_ref, self.star_sep_check) + self.slash_sep_arg <<= attach(self.slash_sep_arg_ref, self.slash_sep_check) + self.slash_sep_vararg <<= attach(self.slash_sep_vararg_ref, self.slash_sep_check) self.endline_semicolon <<= attach(self.endline_semicolon_ref, self.endline_semicolon_check) self.async_stmt <<= attach(self.async_stmt_ref, self.async_stmt_check) self.async_comp_for <<= attach(self.async_comp_for_ref, self.async_comp_check) @@ -1277,8 +1294,8 @@ def match_data_handle(self, original, loc, tokens): matcher = Matcher(loc, match_check_var, name_list=[]) - req_args, def_args, star_arg, kwd_args, dubstar_arg = split_args_list(matches, loc) - matcher.match_function(match_to_args_var, match_to_kwargs_var, req_args + def_args, star_arg, kwd_args, dubstar_arg) + pos_only_args, req_args, def_args, star_arg, kwd_args, dubstar_arg = split_args_list(matches, loc) + matcher.match_function(match_to_args_var, match_to_kwargs_var, pos_only_args, req_args + def_args, star_arg, kwd_args, dubstar_arg) if cond is not None: matcher.add_guard(cond) @@ -1566,8 +1583,8 @@ def name_match_funcdef_handle(self, original, loc, tokens): matcher = Matcher(loc, match_check_var) - req_args, def_args, star_arg, kwd_args, dubstar_arg = split_args_list(matches, loc) - matcher.match_function(match_to_args_var, match_to_kwargs_var, req_args + def_args, star_arg, kwd_args, dubstar_arg) + pos_only_args, req_args, def_args, star_arg, kwd_args, dubstar_arg = split_args_list(matches, loc) + matcher.match_function(match_to_args_var, match_to_kwargs_var, pos_only_args, req_args + def_args, star_arg, kwd_args, dubstar_arg) if cond is not None: matcher.add_guard(cond) @@ -2162,6 +2179,10 @@ def star_sep_check(self, original, loc, tokens): """Check for Python 3 keyword-only arguments.""" return self.check_py("3", "keyword-only argument separator (use 'match' to produce universal code)", original, loc, tokens) + def slash_sep_check(self, original, loc, tokens): + """Check for Python 3.8 positional-only arguments.""" + return self.check_py("38", "positional-only argument separator (use 'match' to produce universal code)", original, loc, tokens) + def matrix_at_check(self, original, loc, tokens): """Check for Python 3.5 matrix multiplication.""" return self.check_py("35", "matrix multiplication", original, loc, tokens) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index dc6cc0b59..422082087 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -978,26 +978,37 @@ class Grammar(object): star_sep_arg_ref = condense(star + arg_comma) star_sep_vararg = Forward() star_sep_vararg_ref = condense(star + vararg_comma) + + slash_sep_arg = Forward() + slash_sep_arg_ref = condense(slash + arg_comma) + slash_sep_vararg = Forward() + slash_sep_vararg_ref = condense(slash + vararg_comma) + just_star = star + rparen + just_slash = slash + rparen + just_op = just_star | just_slash match = Forward() - args_list = ~just_star + trace(addspace(ZeroOrMore(condense( + args_list = ~just_op + trace(addspace(ZeroOrMore(condense( # everything here must end with arg_comma (star | dubstar) + tfpdef | star_sep_arg + | slash_sep_arg | tfpdef_default, )))) parameters = condense(lparen + args_list + rparen) - var_args_list = ~just_star + trace(addspace(ZeroOrMore(condense( + var_args_list = ~just_op + trace(addspace(ZeroOrMore(condense( # everything here must end with vararg_comma (star | dubstar) + name + vararg_comma | star_sep_vararg + | slash_sep_vararg | name + Optional(default) + vararg_comma, )))) match_args_list = trace(Group(Optional(tokenlist( Group( (star | dubstar) + match | star # not star_sep because pattern-matching can handle star separators on any Python version + | slash # not slash_sep as above | match + Optional(equals.suppress() + test), ), comma, @@ -1722,6 +1733,7 @@ class Grammar(object): Group( dubstar - tfpdef_tokens | star - Optional(tfpdef_tokens) + | slash | tfpdef_default_tokens, ) + Optional(passthrough.suppress()), comma + Optional(passthrough), # implicitly suppressed diff --git a/coconut/compiler/matching.py b/coconut/compiler/matching.py index e0ea1aea2..ac2af8890 100644 --- a/coconut/compiler/matching.py +++ b/coconut/compiler/matching.py @@ -246,19 +246,19 @@ def check_len_in(self, min_len, max_len, item): else: self.add_check(str(min_len) + " <= _coconut.len(" + item + ") <= " + str(max_len)) - def match_function(self, args, kwargs, match_args=(), star_arg=None, kwd_args=(), dubstar_arg=None): + def match_function(self, args, kwargs, pos_only_match_args=(), match_args=(), star_arg=None, kwd_match_args=(), dubstar_arg=None): """Matches a pattern-matching function.""" - self.match_in_args_kwargs(match_args, args, kwargs, allow_star_args=star_arg is not None) + self.match_in_args_kwargs(pos_only_match_args, match_args, args, kwargs, allow_star_args=star_arg is not None) if star_arg is not None: self.match(star_arg, args + "[" + str(len(match_args)) + ":]") - self.match_in_kwargs(kwd_args, kwargs) + self.match_in_kwargs(kwd_match_args, kwargs) with self.down_a_level(): if dubstar_arg is None: self.add_check("not " + kwargs) else: self.match(dubstar_arg, kwargs) - def match_in_args_kwargs(self, match_args, args, kwargs, allow_star_args=False): + def match_in_args_kwargs(self, pos_only_match_args, match_args, args, kwargs, allow_star_args=False): """Matches against args or kwargs.""" # before everything, pop the FunctionMatchError from context @@ -268,12 +268,15 @@ def match_in_args_kwargs(self, match_args, args, kwargs, allow_star_args=False): req_len = 0 arg_checks = {} to_match = [] # [(move_down, match, against)] - for i, arg in enumerate(match_args): + for i, arg in enumerate(pos_only_match_args + match_args): if isinstance(arg, tuple): (match, default) = arg else: match, default = arg, None - names = get_match_names(match) + if i < len(pos_only_match_args): # faster if arg in pos_only_match_args + names = None + else: + names = get_match_names(match) if default is None: if not names: req_len = i + 1 @@ -322,7 +325,7 @@ def match_in_args_kwargs(self, match_args, args, kwargs, allow_star_args=False): ) to_match.append((True, match, tempvar)) - max_len = None if allow_star_args else len(match_args) + max_len = None if allow_star_args else len(pos_only_match_args) + len(match_args) self.check_len_in(req_len, max_len, args) for i in sorted(arg_checks): lt_check, ge_check = arg_checks[i] diff --git a/coconut/root.py b/coconut/root.py index 9ab4dea7d..2be0e803e 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 31 +DEVELOP = 32 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index 0842677df..4142e1167 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -537,6 +537,15 @@ def main_test(): assert count("derp", None)[5:10] |> list == ["derp"] * 5 assert count("derp", None)[5:] == count("derp", None) assert count("derp", None)[:5] |> list == ["derp"] * 5 + match def f(a, /, b) = a, b + assert f(1, 2) == (1, 2) + assert f(1, b=2) == (1, 2) + try: + f(a=1, b=2) + except MatchError: + assert True + else: + assert False return True def tco_func() = tco_func() diff --git a/tests/src/extras.coco b/tests/src/extras.coco index dce811ee4..e5e7ae541 100644 --- a/tests/src/extras.coco +++ b/tests/src/extras.coco @@ -124,6 +124,7 @@ def test_extras(): setup(target="3.8") assert parse("(a := b)") assert parse("print(a := 1, b := 2)") + assert parse("def f(a, /, b) = a, b") if CoconutKernel is not None: if PY35: asyncio.set_event_loop(asyncio.new_event_loop()) From d6c526e38e4a1c4e3bda6cfa4df26c22709d6261 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 13 Jun 2019 23:21:22 -0700 Subject: [PATCH 118/163] Remove import * from headers Resolves #507. --- coconut/compiler/header.py | 25 ++++++++++++++++--------- coconut/constants.py | 10 ++++++++++ coconut/root.py | 2 +- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 358570e2d..60e442c01 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -29,6 +29,8 @@ default_encoding, template_ext, justify_len, + reserved_prefix, + ignored_imports, ) from coconut.exceptions import internal_assert @@ -180,7 +182,6 @@ class you_need_to_install_trollius: pass "_coconut.functools.partial(_coconut_base_pattern_func, base_func)" if no_tco else "_coconut_back_compose(_coconut_tco, _coconut.functools.partial(_coconut_base_pattern_func, base_func))" ), - comma_tco=", _coconut_tail_call, _coconut_tco" if not no_tco else "", def_prepattern=( r'''def prepattern(base_func): """DEPRECATED: Use addpattern instead.""" @@ -205,8 +206,6 @@ def NamedTuple(name, fields): return _coconut.collections.namedtuple(name, [x for x, t in fields])'''.format(**format_dict), ) - format_dict["underscore_imports"] = "_coconut, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_back_pipe, _coconut_star_pipe, _coconut_back_star_pipe, _coconut_dubstar_pipe, _coconut_back_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert".format(**format_dict) - # ._coconut_tco_func is used in main.coco, so don't remove it # here without replacing its usage there format_dict["def_tco"] = "" if no_tco else '''class _coconut_tail_call{object}: @@ -234,6 +233,16 @@ def tail_call_optimized_func(*args, **kwargs): return tail_call_optimized_func '''.format(**format_dict) + # only do this when asked for to avoid import cycles and performance implications + # we do this instead of import * + import underscore names to avoid ipython errs + if which == "sys" or which.startswith("package"): + from coconut import __coconut__ + format_dict["import_names"] = ", ".join( + varname for varname in dir(__coconut__) + if (not varname.startswith("_") or varname.startswith(reserved_prefix)) + and varname not in ignored_imports + ) + return format_dict, target_startswith, target_info @@ -255,7 +264,7 @@ def getheader(which, target="", use_hash=None, no_tco=False, strict=False): if which == "none": return "" - # initial, __coconut__, package, sys, code, file + # initial, __coconut__, package:n, sys, code, file format_dict, target_startswith, target_info = process_header_args(which, target, use_hash, no_tco, strict) @@ -274,7 +283,7 @@ def getheader(which, target="", use_hash=None, no_tco=False, strict=False): if which == "initial": return header - # __coconut__, package, sys, code, file + # __coconut__, package:n, sys, code, file header += section("Coconut Header") @@ -296,8 +305,7 @@ def getheader(which, target="", use_hash=None, no_tco=False, strict=False): if _coconut_cached_module is not None and _coconut_os_path.dirname(_coconut_cached_module.__file__) != _coconut_file_path: del _coconut_sys.modules[{__coconut__}] _coconut_sys.path.insert(0, _coconut_file_path) -from __coconut__ import {underscore_imports} -from __coconut__ import * +from __coconut__ import {import_names} {sys_path_pop} '''.format( @@ -319,8 +327,7 @@ def getheader(which, target="", use_hash=None, no_tco=False, strict=False): if which == "sys": return header + '''import sys as _coconut_sys -from coconut.__coconut__ import {underscore_imports} -from coconut.__coconut__ import * +from coconut.__coconut__ import {import_names} '''.format(**format_dict) # __coconut__, code, file diff --git a/coconut/constants.py b/coconut/constants.py index a83eae38e..519551bc5 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -524,6 +524,16 @@ def checksum(data): "asyncio": ("trollius", (3, 4)), } +# names not to import from __coconut__; should include all __future__ imports +ignored_imports = ( + "print_function", + "absolute_import", + "unicode_literals", + "division", + "generator_stop", + "annotations", +) + # ----------------------------------------------------------------------------------------------------------------------- # COMMAND CONSTANTS: # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/root.py b/coconut/root.py index 2be0e803e..6ea4d9f6d 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 32 +DEVELOP = 33 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 0aaddeb36e38bccd57c95fb948750f641eb9aa1a Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 14 Jun 2019 20:22:27 -0700 Subject: [PATCH 119/163] Fix IPython magics Resolves #507. --- coconut/compiler/header.py | 21 ++++++------------ coconut/constants.py | 10 --------- coconut/icoconut/root.py | 45 ++++++++++++++++++++++++++++---------- coconut/root.py | 2 +- 4 files changed, 42 insertions(+), 36 deletions(-) diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 60e442c01..551e7a938 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -29,8 +29,6 @@ default_encoding, template_ext, justify_len, - reserved_prefix, - ignored_imports, ) from coconut.exceptions import internal_assert @@ -196,8 +194,11 @@ def pattern_prepender(func): return _coconut.functools.partial(makedata, data_type) ''' if not strict else "" ), + comma_tco=", _coconut_tail_call, _coconut_tco" if not no_tco else "", ) + format_dict["underscore_imports"] = "_coconut, _coconut_MatchError{comma_tco}, _coconut_igetitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_back_pipe, _coconut_star_pipe, _coconut_back_star_pipe, _coconut_dubstar_pipe, _coconut_back_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert".format(**format_dict) + format_dict["import_typing_NamedTuple"] = _indent( "import typing" if target_info >= (3, 6) else '''class typing{object}: @@ -233,16 +234,6 @@ def tail_call_optimized_func(*args, **kwargs): return tail_call_optimized_func '''.format(**format_dict) - # only do this when asked for to avoid import cycles and performance implications - # we do this instead of import * + import underscore names to avoid ipython errs - if which == "sys" or which.startswith("package"): - from coconut import __coconut__ - format_dict["import_names"] = ", ".join( - varname for varname in dir(__coconut__) - if (not varname.startswith("_") or varname.startswith(reserved_prefix)) - and varname not in ignored_imports - ) - return format_dict, target_startswith, target_info @@ -305,7 +296,8 @@ def getheader(which, target="", use_hash=None, no_tco=False, strict=False): if _coconut_cached_module is not None and _coconut_os_path.dirname(_coconut_cached_module.__file__) != _coconut_file_path: del _coconut_sys.modules[{__coconut__}] _coconut_sys.path.insert(0, _coconut_file_path) -from __coconut__ import {import_names} +from __coconut__ import * +from __coconut__ import {underscore_imports} {sys_path_pop} '''.format( @@ -327,7 +319,8 @@ def getheader(which, target="", use_hash=None, no_tco=False, strict=False): if which == "sys": return header + '''import sys as _coconut_sys -from coconut.__coconut__ import {import_names} +from coconut.__coconut__ import * +from coconut.__coconut__ import {underscore_imports} '''.format(**format_dict) # __coconut__, code, file diff --git a/coconut/constants.py b/coconut/constants.py index 519551bc5..a83eae38e 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -524,16 +524,6 @@ def checksum(data): "asyncio": ("trollius", (3, 4)), } -# names not to import from __coconut__; should include all __future__ imports -ignored_imports = ( - "print_function", - "absolute_import", - "unicode_literals", - "division", - "generator_stop", - "annotations", -) - # ----------------------------------------------------------------------------------------------------------------------- # COMMAND CONSTANTS: # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/icoconut/root.py b/coconut/icoconut/root.py index 7d8bcee4c..2fe6b3c84 100644 --- a/coconut/icoconut/root.py +++ b/coconut/icoconut/root.py @@ -25,6 +25,7 @@ from coconut.exceptions import ( CoconutException, CoconutInternalException, + internal_assert, ) from coconut.constants import ( py_syntax_version, @@ -76,6 +77,7 @@ def memoized_parse_block(code): """Memoized version of parse_block.""" + internal_assert(lambda: code not in parse_block_memo.values(), "attempted recompilation of", code) success, result = parse_block_memo.get(code, (None, None)) if success is None: try: @@ -91,11 +93,6 @@ def memoized_parse_block(code): raise result -def memoized_parse_sys(code): - """Memoized version of parse_sys.""" - return COMPILER.header_proc(memoized_parse_block(code), header="sys", initial="none") - - # ----------------------------------------------------------------------------------------------------------------------- # KERNEL: # ----------------------------------------------------------------------------------------------------------------------- @@ -106,10 +103,17 @@ def memoized_parse_sys(code): class CoconutCompiler(CachingCompiler, object): """IPython compiler for Coconut.""" + def __init__(self): + super(CoconutCompiler, self).__init__() + + # we compile the header here to cache the proper __future__ imports + header = COMPILER.getheader("sys") + super(CoconutCompiler, self).__call__(header, "", "exec") + def ast_parse(self, source, *args, **kwargs): """Version of ast_parse that compiles Coconut code first.""" try: - compiled = memoized_parse_sys(source) + compiled = memoized_parse_block(source) except CoconutException as err: raise err.syntax_err() else: @@ -118,13 +122,24 @@ def ast_parse(self, source, *args, **kwargs): def cache(self, code, *args, **kwargs): """Version of cache that compiles Coconut code first.""" try: - compiled = memoized_parse_sys(code) + compiled = memoized_parse_block(code) except CoconutException: traceback.print_exc() return None else: return super(CoconutCompiler, self).cache(compiled, *args, **kwargs) + def __call__(self, source, *args, **kwargs): + """Version of __call__ that compiles Coconut code first.""" + if isinstance(source, (str, bytes)): + try: + compiled = memoized_parse_block(source) + except CoconutException as err: + raise err.syntax_err() + else: + compiled = source + return super(CoconutCompiler, self).__call__(compiled, *args, **kwargs) + class CoconutSplitter(IPythonInputSplitter, object): """IPython splitter for Coconut.""" @@ -154,15 +169,23 @@ def init_instance_attrs(self): super(CoconutShell, self).init_instance_attrs() self.compile = CoconutCompiler() - def init_create_namespaces(self, *args, **kwargs): + def init_create_namespaces(self, user_module=None, user_ns=None): """Version of init_create_namespaces that adds Coconut built-ins to globals.""" - super(CoconutShell, self).init_create_namespaces(*args, **kwargs) - RUNNER.update_vars(self.user_global_ns) + super(CoconutShell, self).init_create_namespaces(user_module, user_ns) + for namespace in self.ns_table.values(): + RUNNER.update_vars(namespace) - def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=None): + def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=True): """Version of run_cell that always uses shell_futures.""" return super(CoconutShell, self).run_cell(raw_cell, store_history, silent, shell_futures=True) + if hasattr(ZMQInteractiveShell, "run_cell_async"): + import asyncio + @asyncio.coroutine + def run_cell_async(self, raw_cell, store_history=False, silent=False, shell_futures=True): + """Version of run_cell_async that always uses shell_futures.""" + return super(CoconutShell, self).run_cell_async(raw_cell, store_history, silent, shell_futures=True) + def user_expressions(self, expressions): """Version of user_expressions that compiles Coconut code first.""" compiled_expressions = {} diff --git a/coconut/root.py b/coconut/root.py index 6ea4d9f6d..edd94ecf5 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 33 +DEVELOP = 34 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From 74c7a1e7c1b8a2e7524a3fa706cc1fb51f7a8b95 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 14 Jun 2019 20:28:48 -0700 Subject: [PATCH 120/163] Improve asyncio usage --- coconut/icoconut/root.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/coconut/icoconut/root.py b/coconut/icoconut/root.py index 2fe6b3c84..08419ca98 100644 --- a/coconut/icoconut/root.py +++ b/coconut/icoconut/root.py @@ -22,6 +22,11 @@ import os import traceback +try: + import asyncio +except ImportError: + asyncio = None + from coconut.exceptions import ( CoconutException, CoconutInternalException, @@ -179,8 +184,7 @@ def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=Tr """Version of run_cell that always uses shell_futures.""" return super(CoconutShell, self).run_cell(raw_cell, store_history, silent, shell_futures=True) - if hasattr(ZMQInteractiveShell, "run_cell_async"): - import asyncio + if asyncio is not None: @asyncio.coroutine def run_cell_async(self, raw_cell, store_history=False, silent=False, shell_futures=True): """Version of run_cell_async that always uses shell_futures.""" From da671fe546b82d5cb4bb84961f068bd377059741 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 14 Jun 2019 20:38:58 -0700 Subject: [PATCH 121/163] Fix addpattern pickling --- coconut/compiler/templates/header.py_template | 2 +- coconut/root.py | 2 +- tests/src/cocotest/agnostic/util.coco | 11 +++++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index ff4588e50..e1061e7ff 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -475,7 +475,7 @@ class _coconut_base_pattern_func{object}: def __repr__(self): return "addpattern(" + _coconut.repr(self.patterns[0]) + ")(*" + _coconut.repr(self.patterns[1:]) + ")" def __reduce__(self): - return (self.__class__, self.patterns) + return (self.__class__, _coconut.tuple(self.patterns)) def addpattern(base_func): """Decorator to add a new case to a pattern-matching function, where the new case is checked last.""" diff --git a/coconut/root.py b/coconut/root.py index edd94ecf5..5edc80ffc 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 34 +DEVELOP = 35 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/cocotest/agnostic/util.coco b/tests/src/cocotest/agnostic/util.coco index f0e945361..4a32b3017 100644 --- a/tests/src/cocotest/agnostic/util.coco +++ b/tests/src/cocotest/agnostic/util.coco @@ -157,19 +157,22 @@ def qsort6(l: int$[]) -> int$[]: def empty_list_base_case([]) = [] -@addpattern(empty_list_base_case) -def qsort7([head] + tail) = +# use separate name for base func for pickle +def _qsort7([head] + tail) = qsort7(left) + [head] + qsort7(right) where: left = [x for x in tail if x <= head] right = [x for x in tail if x > head] +qsort7 = addpattern(empty_list_base_case)(_qsort7) -@addpattern(empty_list_base_case) -def qsort8(l) = +# use separate name for base func for pickle +def _qsort8(l) = qsort8(left) + [mid] + qsort8(right) where: midpt = len(l)//2 mid, rest = l[midpt], l[:midpt] + l[midpt+1:] left = [x for x in rest if x < mid] right = [x for x in rest if x >= mid] +qsort8 = addpattern(empty_list_base_case)(_qsort8) + # Infinite Iterators: def repeat(elem): From 74783e5e3801d380a960a01d82e886345d24736e Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 15 Jun 2019 14:55:01 -0700 Subject: [PATCH 122/163] Fix addpattern tco pickling --- coconut/compiler/grammar.py | 4 +- coconut/compiler/header.py | 20 +++---- coconut/compiler/templates/header.py_template | 17 +++--- coconut/root.py | 2 +- coconut/stubs/__coconut__.pyi | 59 +++++++++++-------- 5 files changed, 53 insertions(+), 49 deletions(-) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 422082087..c034588db 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -683,8 +683,8 @@ def where_stmt_handle(tokens): """Process a where statement.""" internal_assert(len(tokens) == 2, "invalid where statement tokens", tokens) base_stmt, assignment_stmts = tokens - stmts = list(assignment_stmts) + [base_stmt] - return "\n".join(stmts) + "\n" + stmts = list(assignment_stmts) + [base_stmt + "\n"] + return "".join(stmts) # end: HANDLERS diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 551e7a938..b7ac8c21b 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -175,11 +175,15 @@ class you_need_to_install_trollius: pass with ThreadPoolExecutor(cpu_count() * 5)''' if target_info < (3, 5) else '''with ThreadPoolExecutor()''' ), - tail_call_last_pattern="self.patterns[-1](*args, **kwargs)" if no_tco else "_coconut_tail_call(self.patterns[-1], *args, **kwargs)", - tco_partial_base_pattern_func=( - "_coconut.functools.partial(_coconut_base_pattern_func, base_func)" if no_tco - else "_coconut_back_compose(_coconut_tco, _coconut.functools.partial(_coconut_base_pattern_func, base_func))" - ), + def_tco_func="""def _coconut_tco_func(self, *args, **kwargs): + for func in self.patterns[:-1]: + try: + with _coconut_FunctionMatchErrorContext(self.FunctionMatchError): + return func(*args, **kwargs) + except self.FunctionMatchError: + pass + return _coconut_tail_call(self.patterns[-1], *args, **kwargs) + """, def_prepattern=( r'''def prepattern(base_func): """DEPRECATED: Use addpattern instead.""" @@ -213,15 +217,12 @@ def NamedTuple(name, fields): __slots__ = ("func", "args", "kwargs") def __init__(self, func, *args, **kwargs): self.func, self.args, self.kwargs = func, args, kwargs -_coconut_tco_func_dict = {empty_dict} def _coconut_tco(func): @_coconut.functools.wraps(func) def tail_call_optimized_func(*args, **kwargs): call_func = func while True: - wkref = _coconut_tco_func_dict.get(_coconut.id(call_func)) - if wkref is not None and wkref() is call_func: - call_func = call_func._coconut_tco_func + call_func = _coconut.getattr(call_func, "_coconut_tco_func", call_func) result = call_func(*args, **kwargs) # pass --no-tco to clean up your traceback if not isinstance(result, _coconut_tail_call): return result @@ -230,7 +231,6 @@ def tail_call_optimized_func(*args, **kwargs): tail_call_optimized_func.__module__ = _coconut.getattr(func, "__module__", None) tail_call_optimized_func.__name__ = _coconut.getattr(func, "__name__", "") tail_call_optimized_func.__qualname__ = _coconut.getattr(func, "__qualname__", tail_call_optimized_func.__name__) - _coconut_tco_func_dict[_coconut.id(tail_call_optimized_func)] = _coconut.weakref.ref(tail_call_optimized_func) return tail_call_optimized_func '''.format(**format_dict) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index e1061e7ff..9465e8dac 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -1,5 +1,5 @@ -class _coconut{object}:{comment.everything_here_must_be_copied_to_stub_file} - import collections, copy, functools, types, itertools, operator, weakref, threading +class _coconut{object}:{comment.EVERYTHING_HERE_MUST_BE_COPIED_TO_STUB_FILE} + import collections, copy, functools, types, itertools, operator, threading {bind_lru_cache}{import_asyncio}{import_pickle} {import_OrderedDict} {import_collections_abc} @@ -448,18 +448,15 @@ class _coconut_FunctionMatchErrorContext(object): return _coconut_MatchError _coconut_get_function_match_error = _coconut_FunctionMatchErrorContext.get class _coconut_base_pattern_func{object}: + __slots__ = ("FunctionMatchError", "__doc__", "patterns") def __init__(self, *funcs): self.FunctionMatchError = _coconut.type(_coconut_str("MatchError"), (_coconut_MatchError,), {{}}) self.__doc__ = None - self.__name__ = "" - self.__module__ = None self.patterns = [] for func in funcs: self.add(func) def add(self, func): self.__doc__ = _coconut.getattr(func, "__doc__", None) or self.__doc__ - self.__name__ = _coconut.getattr(func, "__name__", None) or self.__name__ - self.__module__ = _coconut.getattr(func, "__module__", None) or self.__module__ if _coconut.isinstance(func, _coconut_base_pattern_func): self.patterns += func.patterns else: @@ -471,15 +468,15 @@ class _coconut_base_pattern_func{object}: return func(*args, **kwargs) except self.FunctionMatchError: pass - return {tail_call_last_pattern} - def __repr__(self): + return self.patterns[-1](*args, **kwargs) + {def_tco_func}def __repr__(self): return "addpattern(" + _coconut.repr(self.patterns[0]) + ")(*" + _coconut.repr(self.patterns[1:]) + ")" def __reduce__(self): return (self.__class__, _coconut.tuple(self.patterns)) def addpattern(base_func): """Decorator to add a new case to a pattern-matching function, where the new case is checked last.""" - return {tco_partial_base_pattern_func} + return _coconut.functools.partial(_coconut_base_pattern_func, base_func) _coconut_addpattern = addpattern {def_prepattern}class _coconut_partial{object}: __slots__ = ("func", "_argdict", "_arglen", "_stargs", "keywords") @@ -525,7 +522,7 @@ _coconut_addpattern = addpattern return _coconut.repr(self.func) + "$(" + ", ".join(args) + ")" def consume(iterable, keep_last=0): """consume(iterable, keep_last) fully exhausts iterable and return the last keep_last elements.""" - return _coconut.collections.deque(iterable, maxlen=keep_last) # fastest way to exhaust an iterator + return _coconut.collections.deque(iterable, maxlen=keep_last) class starmap(_coconut.itertools.starmap): __slots__ = ("func", "iter") if hasattr(_coconut.itertools.starmap, "__doc__"): diff --git a/coconut/root.py b/coconut/root.py index 5edc80ffc..391aefdd8 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 35 +DEVELOP = 36 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/coconut/stubs/__coconut__.pyi b/coconut/stubs/__coconut__.pyi index 559853bcf..06fe1cd57 100644 --- a/coconut/stubs/__coconut__.pyi +++ b/coconut/stubs/__coconut__.pyi @@ -65,24 +65,24 @@ def scan( class _coconut: - typing = _t # The real _coconut doesn't import typing, but we want type-checkers to treat it as if it does - import collections, copy, functools, types, itertools, operator, types, weakref - import pickle - Ellipsis, Exception, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, classmethod, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, map, min, max, next, object, property, range, reversed, set, slice, str, sum, super, tuple, zip, repr = Ellipsis, Exception, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, classmethod, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, map, min, max, next, object, property, range, reversed, set, slice, str, sum, super, tuple, zip, repr + import collections, copy, functools, types, itertools, operator, threading if sys.version_info >= (3, 4): import asyncio else: import trollius as asyncio # type: ignore + import pickle + if sys.version_info >= (2, 7): + OrderedDict = collections.OrderedDict + else: + OrderedDict = dict if sys.version_info < (3, 3): abc = collections else: abc = collections.abc + typing = _t # The real _coconut doesn't import typing, but we want type-checkers to treat it as if it does + Ellipsis, Exception, AttributeError, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, classmethod, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, map, min, max, next, object, property, range, reversed, set, slice, str, sum, super, tuple, type, zip, repr = Ellipsis, Exception, AttributeError, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, classmethod, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, map, min, max, next, object, property, range, reversed, set, slice, str, sum, super, tuple, type, zip, repr if sys.version_info >= (3,): bytearray = bytearray - if sys.version_info >= (2, 7): - OrderedDict = collections.OrderedDict - else: - OrderedDict = dict reduce = _coconut.functools.reduce @@ -104,10 +104,10 @@ _coconut_starmap = starmap parallel_map = concurrent_map = _coconut_map = map -_coconut_sentinel = object() +TYPE_CHECKING = _t.TYPE_CHECKING -TYPE_CHECKING = _t.TYPE_CHECKING +_coconut_sentinel = object() class MatchError(Exception): ... @@ -117,18 +117,39 @@ _coconut_MatchError = MatchError def _coconut_get_function_match_error() -> _t.Type[MatchError]: ... -def _coconut_tco(func: _FUNC) -> _FUNC: ... +def _coconut_tco(func: _FUNC) -> _FUNC: + return func def _coconut_tail_call(func, *args, **kwargs): return func(*args, **kwargs) -def recursive_iterator(func: _ITER_FUNC) -> _ITER_FUNC: ... +def recursive_iterator(func: _ITER_FUNC) -> _ITER_FUNC: + return func + +class _coconut_base_pattern_func: + def __init__(self, *funcs: _t.Callable): ... + def add(self, func: _t.Callable) -> None: ... + def __call__(self, *args, **kwargs) -> _t.Any: ... def addpattern(func: _FUNC) -> _t.Callable[[_FUNC2], _t.Union[_FUNC, _FUNC2]]: ... _coconut_addpattern = prepattern = addpattern +class _coconut_partial: + args: _t.Tuple = ... + keywords: _t.Dict[_t.Text, _t.Any] = ... + def __init__( + self, + func: _t.Callable[..., _T], + argdict: _t.Dict[int, _t.Any], + arglen: int, + *args, + **kwargs, + ) -> None: ... + def __call__(self, *args, **kwargs) -> _T: ... + + @_t.overload def _coconut_igetitem( iterable: _t.Iterable[_T], @@ -258,18 +279,4 @@ def consume( ) -> _t.Iterable[_T]: ... -class _coconut_partial: - args: _t.Tuple = ... - keywords: _t.Dict[_t.Text, _t.Any] = ... - def __init__( - self, - func: _t.Callable[..., _T], - argdict: _t.Dict[int, _t.Any], - arglen: int, - *args, - **kwargs, - ) -> None: ... - def __call__(self, *args, **kwargs) -> _T: ... - - def fmap(func: _t.Callable, obj: _t.Iterable) -> _t.Iterable: ... From 50c4f7782b610dfedb931e764b57ca1e039b5180 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 15 Jun 2019 16:03:34 -0700 Subject: [PATCH 123/163] Fix IPython namespaces --- coconut/command/util.py | 10 ++++++++-- coconut/icoconut/root.py | 10 +++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/coconut/command/util.py b/coconut/command/util.py index dfc0b222a..51c1a2683 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -486,9 +486,15 @@ def handling_errors(self, all_errors_exit=False): if all_errors_exit: self.exit(1) - def update_vars(self, global_vars): + def update_vars(self, global_vars, ignore_vars=None): """Add Coconut built-ins to given vars.""" - global_vars.update(self.vars) + if ignore_vars: + update_vars = self.vars.copy() + for del_var in ignore_vars: + del update_vars[del_var] + else: + update_vars = self.vars + global_vars.update(update_vars) def run(self, code, use_eval=None, path=None, all_errors_exit=False, store=True): """Execute Python code.""" diff --git a/coconut/icoconut/root.py b/coconut/icoconut/root.py index 08419ca98..5cbdee56e 100644 --- a/coconut/icoconut/root.py +++ b/coconut/icoconut/root.py @@ -174,11 +174,11 @@ def init_instance_attrs(self): super(CoconutShell, self).init_instance_attrs() self.compile = CoconutCompiler() - def init_create_namespaces(self, user_module=None, user_ns=None): - """Version of init_create_namespaces that adds Coconut built-ins to globals.""" - super(CoconutShell, self).init_create_namespaces(user_module, user_ns) - for namespace in self.ns_table.values(): - RUNNER.update_vars(namespace) + def init_user_ns(self): + """Version of init_user_ns that adds Coconut built-ins.""" + super(CoconutShell, self).init_user_ns() + RUNNER.update_vars(self.user_ns) + RUNNER.update_vars(self.user_ns_hidden) def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=True): """Version of run_cell that always uses shell_futures.""" From ab9aadf920b3fcb528474742da1156360212b24f Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 18 Jun 2019 15:45:49 -0700 Subject: [PATCH 124/163] Add pure python override --- coconut/constants.py | 2 ++ coconut/requirements.py | 5 +++++ coconut/root.py | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/coconut/constants.py b/coconut/constants.py index a83eae38e..5083850f6 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -105,6 +105,8 @@ def checksum(data): license_name = "Apache 2.0" +pure_python_env_var = "COCONUT_PURE_PYTHON" + # the different categories here are defined in requirements.py, # anything after a colon is ignored but allows different versions # for different categories, and tuples denote the use of environment diff --git a/coconut/requirements.py b/coconut/requirements.py index d5751fec7..ceb72e1e4 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -17,6 +17,7 @@ from coconut.root import * # NOQA +import os import platform from coconut.constants import ( @@ -26,6 +27,7 @@ all_reqs, min_versions, version_strictly, + pure_python_env_var, PYPY, PY34, IPY, @@ -139,6 +141,9 @@ def everything_in(req_dict): get_reqs("dev"), ) +if os.environ.get(pure_python_env_var, "").lower() == "true": + # override necessary for readthedocs + requirements += get_reqs("purepython") if supports_env_markers: # modern method extras[":platform_python_implementation=='CPython'"] = get_reqs("cpython") diff --git a/coconut/root.py b/coconut/root.py index 391aefdd8..dd972ac9c 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 36 +DEVELOP = 37 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From dce725612597b0f86e5dc50703088cbbdb9c6de6 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 18 Jun 2019 15:48:56 -0700 Subject: [PATCH 125/163] Fix pure python env var --- coconut/requirements.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coconut/requirements.py b/coconut/requirements.py index ceb72e1e4..5a719dac9 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -144,7 +144,7 @@ def everything_in(req_dict): if os.environ.get(pure_python_env_var, "").lower() == "true": # override necessary for readthedocs requirements += get_reqs("purepython") -if supports_env_markers: +elif supports_env_markers: # modern method extras[":platform_python_implementation=='CPython'"] = get_reqs("cpython") extras[":platform_python_implementation!='CPython'"] = get_reqs("purepython") @@ -180,7 +180,7 @@ def everything_in(req_dict): def all_versions(req): """Get all versions of req from PyPI.""" - import requests + import requests # expensive url = "https://pypi.python.org/pypi/" + get_base_req(req) + "/json" return tuple(requests.get(url).json()["releases"].keys()) From c7abf3266ddce41060e485cca33846f85dd960e3 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 18 Jun 2019 16:48:21 -0700 Subject: [PATCH 126/163] Improve COCONUT_PURE_PYTHON support --- DOCS.md | 8 ++++++-- coconut/_pyparsing.py | 6 ++++++ coconut/constants.py | 1 + coconut/requirements.py | 5 ++--- coconut/root.py | 2 +- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/DOCS.md b/DOCS.md index 2d0c86f97..755bbc884 100644 --- a/DOCS.md +++ b/DOCS.md @@ -33,10 +33,14 @@ which will install Coconut and its required dependencies. _Note: If you have an old version of Coconut installed and you want to upgrade, run `pip install --upgrade coconut` instead._ If you are encountering errors running `pip install coconut`, try instead running +```bash +COCONUT_PURE_PYTHON=TRUE pip install --user --upgrade coconut ``` -pip install --user --no-deps --upgrade coconut pyparsing +in `bash` (UNIX) or +```bash +cmd /c "set COCONUT_PURE_PYTHON=TRUE&& pip install --user --upgrade coconut" ``` -which will force Coconut to use the pure-Python [`pyparsing`](https://github.com/pyparsing/pyparsing) module instead of the faster [`cPyparsing`](https://github.com/evhub/cpyparsing) module. If you are still getting errors, you may want to try [using conda](#using-conda) instead. +in `cmd` (Windows) which will force Coconut to use the pure-Python [`pyparsing`](https://github.com/pyparsing/pyparsing) module instead of the faster [`cPyparsing`](https://github.com/evhub/cpyparsing) module. If you are still getting errors, you may want to try [using conda](#using-conda) instead. If `pip install coconut` works, but you cannot access the `coconut` command, be sure that Coconut's installation location is in your `PATH` environment variable. On UNIX, that is `/usr/local/bin` (without `--user`) or `${HOME}/.local/bin/` (with `--user`). diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index a63e9dc1b..52cb90450 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -19,6 +19,7 @@ from coconut.root import * # NOQA +import os import traceback import functools @@ -30,11 +31,16 @@ ver_str_to_tuple, ver_tuple_to_str, get_next_version, + pure_python_env_var, + PURE_PYTHON, ) # warning: do not name this file cPyparsing or pyparsing or it might collide with the following imports try: + if PURE_PYTHON: + raise ImportError("skipping cPyparsing check due to " + pure_python_env_var + " = " + os.environ.get(pure_python_env_var, "")) + import cPyparsing as _pyparsing from cPyparsing import * # NOQA from cPyparsing import ( # NOQA diff --git a/coconut/constants.py b/coconut/constants.py index 5083850f6..3a73a1201 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -106,6 +106,7 @@ def checksum(data): license_name = "Apache 2.0" pure_python_env_var = "COCONUT_PURE_PYTHON" +PURE_PYTHON = os.environ.get(pure_python_env_var, "").lower() in ["true", "1"] # the different categories here are defined in requirements.py, # anything after a colon is ignored but allows different versions diff --git a/coconut/requirements.py b/coconut/requirements.py index 5a719dac9..eb009ea27 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -17,7 +17,6 @@ from coconut.root import * # NOQA -import os import platform from coconut.constants import ( @@ -27,11 +26,11 @@ all_reqs, min_versions, version_strictly, - pure_python_env_var, PYPY, PY34, IPY, WINDOWS, + PURE_PYTHON, ) # ----------------------------------------------------------------------------------------------------------------------- @@ -141,7 +140,7 @@ def everything_in(req_dict): get_reqs("dev"), ) -if os.environ.get(pure_python_env_var, "").lower() == "true": +if PURE_PYTHON: # override necessary for readthedocs requirements += get_reqs("purepython") elif supports_env_markers: diff --git a/coconut/root.py b/coconut/root.py index dd972ac9c..4832c58da 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 37 +DEVELOP = 38 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From bba906f0fd10e38e0d3a1c73a2566e0f63411fe2 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 18 Jun 2019 18:19:50 -0700 Subject: [PATCH 127/163] Fix tco of bound methods --- coconut/compiler/header.py | 8 ++++++-- coconut/compiler/templates/header.py_template | 2 +- coconut/root.py | 2 +- tests/src/cocotest/agnostic/suite.coco | 1 + tests/src/cocotest/agnostic/util.coco | 1 + 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index b7ac8c21b..b61866a68 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -217,12 +217,15 @@ def NamedTuple(name, fields): __slots__ = ("func", "args", "kwargs") def __init__(self, func, *args, **kwargs): self.func, self.args, self.kwargs = func, args, kwargs +_coconut_tco_func_dict = {empty_dict} def _coconut_tco(func): @_coconut.functools.wraps(func) def tail_call_optimized_func(*args, **kwargs): call_func = func - while True: - call_func = _coconut.getattr(call_func, "_coconut_tco_func", call_func) + while True:{comment.weakrefs_necessary_for_ignoring_bound_methods} + wkref = _coconut_tco_func_dict.get(_coconut.id(call_func)) + if (wkref is not None and wkref() is call_func) or _coconut.isinstance(call_func, _coconut_base_pattern_func): + call_func = call_func._coconut_tco_func result = call_func(*args, **kwargs) # pass --no-tco to clean up your traceback if not isinstance(result, _coconut_tail_call): return result @@ -231,6 +234,7 @@ def tail_call_optimized_func(*args, **kwargs): tail_call_optimized_func.__module__ = _coconut.getattr(func, "__module__", None) tail_call_optimized_func.__name__ = _coconut.getattr(func, "__name__", "") tail_call_optimized_func.__qualname__ = _coconut.getattr(func, "__qualname__", tail_call_optimized_func.__name__) + _coconut_tco_func_dict[_coconut.id(tail_call_optimized_func)] = _coconut.weakref.ref(tail_call_optimized_func) return tail_call_optimized_func '''.format(**format_dict) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 9465e8dac..018377f83 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -1,5 +1,5 @@ class _coconut{object}:{comment.EVERYTHING_HERE_MUST_BE_COPIED_TO_STUB_FILE} - import collections, copy, functools, types, itertools, operator, threading + import collections, copy, functools, types, itertools, operator, threading, weakref {bind_lru_cache}{import_asyncio}{import_pickle} {import_OrderedDict} {import_collections_abc} diff --git a/coconut/root.py b/coconut/root.py index 4832c58da..9a598c11a 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 38 +DEVELOP = 39 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/cocotest/agnostic/suite.coco b/tests/src/cocotest/agnostic/suite.coco index ed0a016df..67d6b370e 100644 --- a/tests/src/cocotest/agnostic/suite.coco +++ b/tests/src/cocotest/agnostic/suite.coco @@ -507,6 +507,7 @@ def suite_test(): assert join_pairs2([(1, [2]), (1, [3])]).items() |> list == [(1, [3, 2])] assert return_in_loop(10) assert methtest().meth(5) == 5 + assert methtest().tail_call_meth(3) == 3 def test_match_error_addpattern(x is int): raise MatchError() @addpattern(test_match_error_addpattern) def test_match_error_addpattern(x) = x diff --git a/tests/src/cocotest/agnostic/util.coco b/tests/src/cocotest/agnostic/util.coco index 4a32b3017..5bc3a5769 100644 --- a/tests/src/cocotest/agnostic/util.coco +++ b/tests/src/cocotest/agnostic/util.coco @@ -272,6 +272,7 @@ def return_in_loop(x): class methtest: def meth(self, arg) = meth(self, arg) + def tail_call_meth(self, arg) = self.meth(arg) def meth(self, arg) = arg # Data Blocks: From d6ce1016f433051a6b556cd358ff0fbd9202be74 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 18 Jun 2019 23:08:30 -0700 Subject: [PATCH 128/163] Fix addpattern/composition/recursive_iterator methods --- coconut/compiler/templates/header.py_template | 6 ++++++ coconut/root.py | 2 +- tests/src/cocotest/agnostic/suite.coco | 6 ++++++ tests/src/cocotest/agnostic/util.coco | 16 ++++++++++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 018377f83..3dac78a11 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -51,6 +51,8 @@ class _coconut_base_compose{object}: return _coconut.repr(self.func) + " " + " ".join(("..*> " if star == 1 else "..**>" if star == 2 else "..> ") + _coconut.repr(f) for f, star in self.funcstars) def __reduce__(self): return (self.__class__, (self.func,) + _coconut.tuple(self.funcstars)) + def __get__(self, obj, objtype=None): + return _coconut.functools.partial(self, obj) def _coconut_forward_compose(func, *funcs): return _coconut_base_compose(func, *((f, 0) for f in funcs)) def _coconut_back_compose(*funcs): return _coconut_forward_compose(*_coconut.reversed(funcs)) def _coconut_forward_star_compose(func, *funcs): return _coconut_base_compose(func, *((f, 1) for f in funcs)) @@ -423,6 +425,8 @@ class recursive_iterator{object}: return "@recursive_iterator(" + _coconut.repr(self.func) + ")" def __reduce__(self): return (self.__class__, (self.func,)) + def __get__(self, obj, objtype=None): + return _coconut.functools.partial(self, obj) class _coconut_FunctionMatchErrorContext(object): __slots__ = ('exc_class', 'taken') threadlocal_var = _coconut.threading.local() @@ -473,6 +477,8 @@ class _coconut_base_pattern_func{object}: return "addpattern(" + _coconut.repr(self.patterns[0]) + ")(*" + _coconut.repr(self.patterns[1:]) + ")" def __reduce__(self): return (self.__class__, _coconut.tuple(self.patterns)) + def __get__(self, obj, objtype=None): + return _coconut.functools.partial(self, obj) def addpattern(base_func): """Decorator to add a new case to a pattern-matching function, where the new case is checked last.""" diff --git a/coconut/root.py b/coconut/root.py index 9a598c11a..de81052b7 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 39 +DEVELOP = 40 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/cocotest/agnostic/suite.coco b/tests/src/cocotest/agnostic/suite.coco index 67d6b370e..a0b61ee42 100644 --- a/tests/src/cocotest/agnostic/suite.coco +++ b/tests/src/cocotest/agnostic/suite.coco @@ -81,6 +81,8 @@ def suite_test(): assert not vector(1, 2) == (1, 2) assert not vector(2, 3) != vector(2, 3) assert vector(1, 2) != (1, 2) + assert vector(1, 2) + vector(2, 3) == vector(3, 5) + assert vector(1, 2) + 1 == vector(2, 3) assert triangle(3, 4, 5).is_right() assert (.)(triangle(3, 4, 5), "is_right") assert (.is_right())(triangle(3, 4, 5)) @@ -561,6 +563,10 @@ def suite_test(): assert False assert issubclass(data6, BaseClass) assert namedpt("a", 3, 4).mag() == 5 + t = descriptor_test() + assert t.lam() == t + assert t.comp() == (t,) + assert t.N()$[:2] |> list == [(t, 0), (t, 1)] return True def tco_test(): diff --git a/tests/src/cocotest/agnostic/util.coco b/tests/src/cocotest/agnostic/util.coco index 5bc3a5769..e5fc90cab 100644 --- a/tests/src/cocotest/agnostic/util.coco +++ b/tests/src/cocotest/agnostic/util.coco @@ -298,6 +298,10 @@ data vector(x, y): return vector(self.x + x, self.y + y) else: raise TypeError() + def __add__(self, vector(x_, y_)) = + vector(self.x + x_, self.y + y_) + addpattern def __add__(self, n is int) = + vector(self.x + n, self.y + n) data triangle(a, b, c): def is_right(self): return self.a**2 + self.b**2 == self.c**2 @@ -908,3 +912,15 @@ data data6(x is int) from BaseClass data namedpt(name is str, x is int, y is int): def mag(self) = (self.x**2 + self.y**2)**0.5 + + +# Descriptor test +def tuplify(*args) = args + +class descriptor_test: + lam = self -> self + comp = tuplify .. ident + + @recursive_iterator + def N(self, i=0) = + [(self, i)] :: self.N(i+1) From 6a04df2408c032e34000243fef57fa3ba253c802 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 19 Jun 2019 11:52:54 -0700 Subject: [PATCH 129/163] Fix broken tests --- tests/main_test.py | 2 +- tests/src/cocotest/agnostic/util.coco | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/main_test.py b/tests/main_test.py index 55c3a2b83..cddd6b964 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -316,7 +316,7 @@ def comp_prelude(args=[], **kwargs): call(["git", "clone", prelude_git]) call_coconut([os.path.join(prelude, "setup.coco"), "--strict"] + args, **kwargs) if PY36: - args.append("--target", "3.6", "--mypy") + args.extend(["--target", "3.6", "--mypy"]) call_coconut([os.path.join(prelude, "prelude-source"), os.path.join(prelude, "prelude"), "--strict"] + args, **kwargs) diff --git a/tests/src/cocotest/agnostic/util.coco b/tests/src/cocotest/agnostic/util.coco index e5fc90cab..9c2b01a4d 100644 --- a/tests/src/cocotest/agnostic/util.coco +++ b/tests/src/cocotest/agnostic/util.coco @@ -300,7 +300,7 @@ data vector(x, y): raise TypeError() def __add__(self, vector(x_, y_)) = vector(self.x + x_, self.y + y_) - addpattern def __add__(self, n is int) = + addpattern def __add__(self, n is int) = # type: ignore vector(self.x + n, self.y + n) data triangle(a, b, c): def is_right(self): From d1b03f541ceeffa54e52766469e5b79fbff6f12d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 19 Jun 2019 16:10:46 -0700 Subject: [PATCH 130/163] Fix more failing tests --- Makefile | 8 ++++---- tests/src/extras.coco | 3 --- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 230fe89bf..aed7fa2d6 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ test-all: # for quickly testing nearly everything locally, just use test-basic .PHONY: test-basic test-basic: - python ./tests --force + python ./tests --strict --force python ./tests/dest/runner.py python ./tests/dest/extras.py @@ -30,21 +30,21 @@ test-basic: # should only be used when testing the tests not the compiler .PHONY: test-tests test-tests: - python ./tests + python ./tests --strict python ./tests/dest/runner.py python ./tests/dest/extras.py # same as test-basic but also runs mypy .PHONY: test-mypy test-mypy: - python ./tests --force --target sys --mypy --follow-imports silent --ignore-missing-imports + python ./tests --strict --force --target sys --mypy --follow-imports silent --ignore-missing-imports python ./tests/dest/runner.py python ./tests/dest/extras.py # same as test-basic but includes verbose output for better debugging .PHONY: test-verbose test-verbose: - python ./tests --force --verbose --jobs 0 + python ./tests --strict --force --verbose --jobs 0 python ./tests/dest/runner.py python ./tests/dest/extras.py diff --git a/tests/src/extras.coco b/tests/src/extras.coco index e5e7ae541..662163eba 100644 --- a/tests/src/extras.coco +++ b/tests/src/extras.coco @@ -1,6 +1,3 @@ -import sys -import os - from coconut.__coconut__ import consume as coc_consume # type: ignore from coconut.constants import ( From 33d53077575f0813c728f9c85d53367286444496 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 19 Jun 2019 17:51:05 -0700 Subject: [PATCH 131/163] Add FUNDING.yml --- FUNDING.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 FUNDING.yml diff --git a/FUNDING.yml b/FUNDING.yml new file mode 100644 index 000000000..e7961dedf --- /dev/null +++ b/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +# github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +# patreon: # Replace with a single Patreon username +open_collective: coconut +# ko_fi: # Replace with a single Ko-fi username +# tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +# community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +# liberapay: # Replace with a single Liberapay username +# issuehunt: # Replace with a single IssueHunt username +# otechie: # Replace with a single Otechie username +# custom: # Replace with a single custom sponsorship URL From 0525b7d622f08e54350fb4749d98dafb2c00cbd5 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 29 Jun 2019 18:23:18 -0700 Subject: [PATCH 132/163] Add coconut_eval --- DOCS.md | 6 ++++++ coconut/convenience.py | 6 +++++- tests/src/extras.coco | 2 ++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/DOCS.md b/DOCS.md index 755bbc884..0d435610b 100644 --- a/DOCS.md +++ b/DOCS.md @@ -2465,6 +2465,12 @@ while True: Executes the given _args_ as if they were fed to `coconut` on the command-line, with the exception that unless _interact_ is true or `-i` is passed, the interpreter will not be started. Additionally, since `parse` and `cmd` share the same convenience parsing object, any changes made to the parsing with `cmd` will work just as if they were made with `setup`. +#### `coconut_eval` + +**coconut.convenience.coconut_eval**(_expression_, _globals_=`None`, _locals_=`None`) + +Version of [`eval`](https://docs.python.org/3/library/functions.html#eval) which can evaluate Coconut code. Uses the same convenience parsing object as the other functions and thus can be controlled by `setup`. + #### `version` **coconut.convenience.version**(**[**_which_**]**) diff --git a/coconut/convenience.py b/coconut/convenience.py index cbd6a5aa4..fd33f7f39 100644 --- a/coconut/convenience.py +++ b/coconut/convenience.py @@ -97,6 +97,11 @@ def parse(code="", mode="sys"): ) +def coconut_eval(expression, globals=None, locals=None): + """Compile and evaluate Coconut code.""" + return eval(parse(expression, "eval"), globals, locals) + + # ----------------------------------------------------------------------------------------------------------------------- # IMPORTER: # ----------------------------------------------------------------------------------------------------------------------- @@ -133,7 +138,6 @@ def find_module(self, fullname, path=None): self.run_compiler(path) # Coconut package was found and compiled, now let Python import it return - return coconut_importer = CoconutImporter() diff --git a/tests/src/extras.coco b/tests/src/extras.coco index 662163eba..c56323c4d 100644 --- a/tests/src/extras.coco +++ b/tests/src/extras.coco @@ -19,6 +19,7 @@ from coconut.convenience import ( version, setup, parse, + coconut_eval, ) if IPY and not WINDOWS: @@ -122,6 +123,7 @@ def test_extras(): assert parse("(a := b)") assert parse("print(a := 1, b := 2)") assert parse("def f(a, /, b) = a, b") + assert coconut_eval("x -> x + 1")(2) == 3 if CoconutKernel is not None: if PY35: asyncio.set_event_loop(asyncio.new_event_loop()) From 636750943ec83b1756e51cd8fd7452ad5b668bb2 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 29 Jun 2019 18:29:09 -0700 Subject: [PATCH 133/163] Fix coconut_eval --- coconut/command/command.py | 4 ++-- coconut/convenience.py | 4 ++++ tests/src/extras.coco | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index 8d3424e56..e2b0468ad 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -578,9 +578,9 @@ def execute_file(self, destpath): self.check_runner() self.runner.run_file(destpath) - def check_runner(self): + def check_runner(self, fix_path=True): """Make sure there is a runner.""" - if os.getcwd() not in sys.path: + if fix_path and s.getcwd() not in sys.path: sys.path.append(os.getcwd()) if self.runner is None: self.runner = Runner(self.comp, exit=self.exit_runner, store=self.mypy) diff --git a/coconut/convenience.py b/coconut/convenience.py index fd33f7f39..23b14c20a 100644 --- a/coconut/convenience.py +++ b/coconut/convenience.py @@ -99,6 +99,10 @@ def parse(code="", mode="sys"): def coconut_eval(expression, globals=None, locals=None): """Compile and evaluate Coconut code.""" + CLI.check_runner(fix_path=False) + if globals is None: + globals = {} + CLI.runner.update_vars(globals) return eval(parse(expression, "eval"), globals, locals) diff --git a/tests/src/extras.coco b/tests/src/extras.coco index c56323c4d..120ef4a34 100644 --- a/tests/src/extras.coco +++ b/tests/src/extras.coco @@ -124,6 +124,7 @@ def test_extras(): assert parse("print(a := 1, b := 2)") assert parse("def f(a, /, b) = a, b") assert coconut_eval("x -> x + 1")(2) == 3 + assert coconut_eval("addpattern") if CoconutKernel is not None: if PY35: asyncio.set_event_loop(asyncio.new_event_loop()) From d0a0f212c7802e1b123d0ea63cbdef98c70176b7 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 29 Jun 2019 23:56:01 -0700 Subject: [PATCH 134/163] Fix typo --- coconut/command/command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index e2b0468ad..66700203d 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -580,7 +580,7 @@ def execute_file(self, destpath): def check_runner(self, fix_path=True): """Make sure there is a runner.""" - if fix_path and s.getcwd() not in sys.path: + if fix_path and os.getcwd() not in sys.path: sys.path.append(os.getcwd()) if self.runner is None: self.runner = Runner(self.comp, exit=self.exit_runner, store=self.mypy) From bd04ed4adffc5831234e28d0a0b8ffc56d93c735 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 30 Jun 2019 00:57:08 -0700 Subject: [PATCH 135/163] Fix coconut_eval --- coconut/command/command.py | 4 ++-- coconut/convenience.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index 66700203d..b8d74208a 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -578,9 +578,9 @@ def execute_file(self, destpath): self.check_runner() self.runner.run_file(destpath) - def check_runner(self, fix_path=True): + def check_runner(self, set_up_path=True): """Make sure there is a runner.""" - if fix_path and os.getcwd() not in sys.path: + if set_up_path and os.getcwd() not in sys.path: sys.path.append(os.getcwd()) if self.runner is None: self.runner = Runner(self.comp, exit=self.exit_runner, store=self.mypy) diff --git a/coconut/convenience.py b/coconut/convenience.py index 23b14c20a..1e63aa38b 100644 --- a/coconut/convenience.py +++ b/coconut/convenience.py @@ -99,7 +99,9 @@ def parse(code="", mode="sys"): def coconut_eval(expression, globals=None, locals=None): """Compile and evaluate Coconut code.""" - CLI.check_runner(fix_path=False) + if CLI.comp is None: + setup() + CLI.check_runner(set_up_path=False) if globals is None: globals = {} CLI.runner.update_vars(globals) From db05ddde204debd6f689bc1f1b35c2499713102d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 30 Jun 2019 01:06:28 -0700 Subject: [PATCH 136/163] Fix test-verbose --- coconut/convenience.py | 5 ++--- tests/__main__.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/coconut/convenience.py b/coconut/convenience.py index 1e63aa38b..0c296e72d 100644 --- a/coconut/convenience.py +++ b/coconut/convenience.py @@ -88,13 +88,12 @@ def parse(code="", mode="sys"): """Compile Coconut code.""" if CLI.comp is None: setup() - if mode in PARSERS: - return PARSERS[mode](CLI.comp)(code) - else: + if mode not in PARSERS: raise CoconutException( "invalid parse mode " + ascii(mode), extra="valid modes are " + ", ".join(PARSERS), ) + return PARSERS[mode](CLI.comp)(code) def coconut_eval(expression, globals=None, locals=None): diff --git a/tests/__main__.py b/tests/__main__.py index 931ee6331..8023f2a3e 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -35,7 +35,7 @@ def main(args=None): if args is None: args = sys.argv[1:] print("Compiling Coconut test suite with args %r." % args) - comp_all(args, expect_retcode=0 if "--mypy" not in args else None) + comp_all(args, expect_retcode=0 if "--mypy" not in args else None, check_errors="--verbose" not in args) if __name__ == "__main__": From 62bb7a89499d5ba68e6065ad96bac72234aeb3ef Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 30 Jun 2019 14:46:03 -0700 Subject: [PATCH 137/163] Fix coconut_eval test --- tests/src/extras.coco | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/src/extras.coco b/tests/src/extras.coco index 120ef4a34..5c4531069 100644 --- a/tests/src/extras.coco +++ b/tests/src/extras.coco @@ -61,7 +61,8 @@ def test_extras(): assert version("-v") assert_raises(-> version("other"), CoconutException) assert_raises(def -> raise CoconutException("derp").syntax_err(), SyntaxError) - setup() + assert coconut_eval("x -> x + 1")(2) == 3 + assert coconut_eval("addpattern") assert parse("abc") == parse("abc", "sys") assert parse("abc", "file") assert parse("abc", "package") @@ -123,8 +124,6 @@ def test_extras(): assert parse("(a := b)") assert parse("print(a := 1, b := 2)") assert parse("def f(a, /, b) = a, b") - assert coconut_eval("x -> x + 1")(2) == 3 - assert coconut_eval("addpattern") if CoconutKernel is not None: if PY35: asyncio.set_event_loop(asyncio.new_event_loop()) From 7b1379a1b0e21b89b978a47e006b5d3df9b1fc3d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 3 Jul 2019 19:25:19 -0700 Subject: [PATCH 138/163] Add Easter egg --- coconut/compiler/compiler.py | 22 +++++++++++++ coconut/compiler/grammar.py | 6 ++-- coconut/compiler/templates/header.py_template | 2 +- coconut/root.py | 2 +- tests/src/cocotest/agnostic/main.coco | 31 +++++++++++++------ 5 files changed, 49 insertions(+), 14 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index b1fa0673d..2d47db426 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -255,6 +255,22 @@ def imported_names(imports): yield imp[-1].split(".", 1)[0] +def special_starred_import_handle(imp_all=False): + """Handles the [from *] import * Coconut Easter egg.""" + if imp_all: + return """ + for _coconut_m in _coconut_sys.modules.values():{oind} + for _coconut_k, _coconut_v in _coconut_m.__dict__.items():{oind} + if not _coconut_k.startswith("_"):{oind} + _coconut.locals()[_coconut_k] = _coconut_v{cind}{cind}{cind} + """.strip().format(oind=openindent, cind=closeindent) + else: + return """ + for _coconut_n, _coconut_m in _coconut_sys.modules.items():{oind} + _coconut.locals()[_coconut_n] = _coconut_m{cind} + """.strip().format(oind=openindent, cind=closeindent) + + def split_args_list(tokens, loc): """Splits function definition arguments.""" pos_only_args = [] @@ -1520,6 +1536,12 @@ def import_handle(self, original, loc, tokens): return "" else: raise CoconutInternalException("invalid import tokens", tokens) + imports = list(imports) + if imp_from == "*" or imp_from is None and "*" in imports: + logger.warn_err(self.make_err(CoconutSyntaxWarning, "[from *] import * is a Coconut Easter egg and should not be used in production code", original, loc)) + if not (len(imports) == 1 and imports[0] == "*"): + raise self.make_err(CoconutSyntaxError, "only [from *] import * allowed, not from * import name", original, loc) + return special_starred_import_handle(imp_all=bool(imp_from)) if self.strict: self.unused_imports.update(imported_names(imports)) return universal_import(imports, imp_from=imp_from, target=self.target) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index c034588db..afeef484f 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -1360,10 +1360,10 @@ class Grammar(object): import_as_name = Group(name - Optional(keyword("as").suppress() - name)) import_names = Group(maybeparens(lparen, tokenlist(dotted_as_name, comma), rparen)) from_import_names = Group(maybeparens(lparen, tokenlist(import_as_name, comma), rparen)) - basic_import = keyword("import").suppress() - import_names + basic_import = keyword("import").suppress() - (import_names | Group(star)) from_import = (keyword("from").suppress() - - condense(ZeroOrMore(unsafe_dot) + dotted_name | OneOrMore(unsafe_dot)) - - keyword("import").suppress() - (Group(star) | from_import_names)) + - condense(ZeroOrMore(unsafe_dot) + dotted_name | OneOrMore(unsafe_dot) | star) + - keyword("import").suppress() - (from_import_names | Group(star))) import_stmt = Forward() import_stmt_ref = from_import | basic_import diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 3dac78a11..9c89d659d 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -4,7 +4,7 @@ class _coconut{object}:{comment.EVERYTHING_HERE_MUST_BE_COPIED_TO_STUB_FILE} {import_OrderedDict} {import_collections_abc} {import_typing_NamedTuple} - Ellipsis, Exception, AttributeError, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, classmethod, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, map, min, max, next, object, property, range, reversed, set, slice, str, sum, super, tuple, type, zip, repr{comma_bytearray} = Ellipsis, Exception, AttributeError, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, classmethod, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, map, min, max, next, object, property, range, reversed, set, slice, str, sum, super, tuple, type, zip, {static_repr}{comma_bytearray} + Ellipsis, Exception, AttributeError, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, classmethod, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, locals, map, min, max, next, object, property, range, reversed, set, slice, str, sum, super, tuple, type, zip, repr{comma_bytearray} = Ellipsis, Exception, AttributeError, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, classmethod, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, locals, map, min, max, next, object, property, range, reversed, set, slice, str, sum, super, tuple, type, zip, {static_repr}{comma_bytearray} _coconut_sentinel = _coconut.object() class MatchError(Exception): """Pattern-matching error. Has attributes .pattern and .value.""" diff --git a/coconut/root.py b/coconut/root.py index de81052b7..6eb6e05c4 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 40 +DEVELOP = 41 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index 4142e1167..124026b1c 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -2,6 +2,15 @@ import sys import asyncio # type: ignore +def assert_raises(c, exc): + """Test whether callable c raises an exception of type exc.""" + try: + c() + except exc: + return True + else: + raise AssertionError("%r failed to raise exception %r" % (c, exc)) + def main_test(): """Basic no-dependency tests.""" assert "\n" == ( @@ -548,16 +557,17 @@ def main_test(): assert False return True -def tco_func() = tco_func() +def easter_egg_test(): + import sys as _sys + import * + assert sys == _sys + orig_name = __name__ + from * import * + assert __name__ == orig_name + assert locals()["byteorder"] == _sys.byteorder + return True -def assert_raises(c, exc): - """Test whether callable c raises an exception of type exc.""" - try: - c() - except exc: - return True - else: - raise AssertionError("%r failed to raise exception %r" % (c, exc)) +def tco_func() = tco_func() def main(*args): """Asserts arguments and executes tests.""" @@ -604,4 +614,7 @@ def main(*args): print(".", end="") # ........ from . import tutorial # type: ignore + print(".", end="") # ......... + assert easter_egg_test() + print("\n") From 6efa79296c46ecb450bd58ede26e4e7c7ace8218 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 3 Jul 2019 20:40:01 -0700 Subject: [PATCH 139/163] Fix Easter egg --- coconut/compiler/compiler.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 2d47db426..b2eab9992 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -260,9 +260,11 @@ def special_starred_import_handle(imp_all=False): if imp_all: return """ for _coconut_m in _coconut_sys.modules.values():{oind} - for _coconut_k, _coconut_v in _coconut_m.__dict__.items():{oind} - if not _coconut_k.startswith("_"):{oind} - _coconut.locals()[_coconut_k] = _coconut_v{cind}{cind}{cind} + _coconut_d = _coconut.getattr(_coconut_m, "__dict__") + if _coconut_d is not None:{oind} + for _coconut_k, _coconut_v in _coconut_d.items():{oind} + if not _coconut_k.startswith("_"):{oind} + _coconut.locals()[_coconut_k] = _coconut_v{cind}{cind}{cind}{cind} """.strip().format(oind=openindent, cind=closeindent) else: return """ From b4df4a81e6561f2d41a152534aacaf8134c08685 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 8 Jul 2019 18:48:56 -0700 Subject: [PATCH 140/163] Further fix Easter egg --- coconut/compiler/compiler.py | 33 ++++++++++++++++--- coconut/compiler/templates/header.py_template | 2 +- coconut/root.py | 2 +- tests/src/cocotest/agnostic/main.coco | 2 ++ 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index b2eab9992..7b85ea0c3 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -257,20 +257,43 @@ def imported_names(imports): def special_starred_import_handle(imp_all=False): """Handles the [from *] import * Coconut Easter egg.""" + out = """ + import imp as _coconut_imp + for _coconut_base_path in _coconut_sys.path:{oind} + for _coconut_dirpath, _coconut_dirnames, _coconut_filenames in _coconut.os.walk(_coconut_base_path):{oind} + _coconut_paths_to_imp = [] + for _coconut_fname in _coconut_filenames:{oind} + if _coconut.os.path.splitext(_coconut_fname)[-1] == "py":{oind} + _coconut_fpath = _coconut.os.path.join(_coconut_dirpath, _coconut_fname) + _coconut_paths_to_imp.append(_coconut_fpath){cind}{cind} + for _coconut_dname in _coconut_dirnames:{oind} + _coconut_dpath = _coconut.os.path.join(_coconut_dirpath, _coconut_dname) + if "__init__.py" in _coconut.os.listdir(_coconut_dpath):{oind} + _coconut_paths_to_imp.append(_coconut_dpath){cind}{cind} + for _coconut_imp_path in _coconut_paths_to_imp:{oind} + _coconut_imp_name = _coconut.os.path.splitext(_coconut.os.path.basename(_coconut_imp_path))[0] + descr = _coconut_imp.find_module(_coconut_imp_name, [_coconut.os.path.dirname(_coconut_imp_path)]) + try:{oind} + _coconut_imp.load_module(_coconut_imp_name, *descr){cind} + except:{oind} + pass{cind}{cind} + _coconut_dirnames[:] = []{cind}{cind} + """.strip().format(oind=openindent, cind=closeindent) if imp_all: - return """ - for _coconut_m in _coconut_sys.modules.values():{oind} - _coconut_d = _coconut.getattr(_coconut_m, "__dict__") + out += "\n" + """ + for _coconut_m in _coconut.tuple(_coconut_sys.modules.values()):{oind} + _coconut_d = _coconut.getattr(_coconut_m, "__dict__", None) if _coconut_d is not None:{oind} for _coconut_k, _coconut_v in _coconut_d.items():{oind} if not _coconut_k.startswith("_"):{oind} _coconut.locals()[_coconut_k] = _coconut_v{cind}{cind}{cind}{cind} """.strip().format(oind=openindent, cind=closeindent) else: - return """ - for _coconut_n, _coconut_m in _coconut_sys.modules.items():{oind} + out += "\n" + """ + for _coconut_n, _coconut_m in _coconut.tuple(_coconut_sys.modules.items()):{oind} _coconut.locals()[_coconut_n] = _coconut_m{cind} """.strip().format(oind=openindent, cind=closeindent) + return out def split_args_list(tokens, loc): diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 9c89d659d..f6a52d7ec 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -1,5 +1,5 @@ class _coconut{object}:{comment.EVERYTHING_HERE_MUST_BE_COPIED_TO_STUB_FILE} - import collections, copy, functools, types, itertools, operator, threading, weakref + import collections, copy, functools, types, itertools, operator, threading, weakref, os {bind_lru_cache}{import_asyncio}{import_pickle} {import_OrderedDict} {import_collections_abc} diff --git a/coconut/root.py b/coconut/root.py index 6eb6e05c4..a1934f975 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 41 +DEVELOP = 42 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index 124026b1c..d98ffc524 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -559,8 +559,10 @@ def main_test(): def easter_egg_test(): import sys as _sys + num_mods_0 = len(_sys.modules) import * assert sys == _sys + assert len(_sys.modules) > num_mods_0 orig_name = __name__ from * import * assert __name__ == orig_name From c2df1b206ec8f5854ccda7677ab02e74aa4bbbb0 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 8 Jul 2019 19:21:39 -0700 Subject: [PATCH 141/163] Improve indentation handling --- coconut/compiler/compiler.py | 231 ++++++++++++++++++----------------- coconut/compiler/util.py | 27 ++++ 2 files changed, 149 insertions(+), 109 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 7b85ea0c3..adbe701b9 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -118,6 +118,7 @@ keyword, append_it, interleaved_join, + handle_indentation, ) from coconut.compiler.header import ( minify, @@ -257,42 +258,42 @@ def imported_names(imports): def special_starred_import_handle(imp_all=False): """Handles the [from *] import * Coconut Easter egg.""" - out = """ - import imp as _coconut_imp - for _coconut_base_path in _coconut_sys.path:{oind} - for _coconut_dirpath, _coconut_dirnames, _coconut_filenames in _coconut.os.walk(_coconut_base_path):{oind} - _coconut_paths_to_imp = [] - for _coconut_fname in _coconut_filenames:{oind} - if _coconut.os.path.splitext(_coconut_fname)[-1] == "py":{oind} - _coconut_fpath = _coconut.os.path.join(_coconut_dirpath, _coconut_fname) - _coconut_paths_to_imp.append(_coconut_fpath){cind}{cind} - for _coconut_dname in _coconut_dirnames:{oind} - _coconut_dpath = _coconut.os.path.join(_coconut_dirpath, _coconut_dname) - if "__init__.py" in _coconut.os.listdir(_coconut_dpath):{oind} - _coconut_paths_to_imp.append(_coconut_dpath){cind}{cind} - for _coconut_imp_path in _coconut_paths_to_imp:{oind} - _coconut_imp_name = _coconut.os.path.splitext(_coconut.os.path.basename(_coconut_imp_path))[0] - descr = _coconut_imp.find_module(_coconut_imp_name, [_coconut.os.path.dirname(_coconut_imp_path)]) - try:{oind} - _coconut_imp.load_module(_coconut_imp_name, *descr){cind} - except:{oind} - pass{cind}{cind} - _coconut_dirnames[:] = []{cind}{cind} - """.strip().format(oind=openindent, cind=closeindent) + out = handle_indentation(""" +import imp as _coconut_imp +for _coconut_base_path in _coconut_sys.path: + for _coconut_dirpath, _coconut_dirnames, _coconut_filenames in _coconut.os.walk(_coconut_base_path): + _coconut_paths_to_imp = [] + for _coconut_fname in _coconut_filenames: + if _coconut.os.path.splitext(_coconut_fname)[-1] == "py": + _coconut_fpath = _coconut.os.path.join(_coconut_dirpath, _coconut_fname) + _coconut_paths_to_imp.append(_coconut_fpath) + for _coconut_dname in _coconut_dirnames: + _coconut_dpath = _coconut.os.path.join(_coconut_dirpath, _coconut_dname) + if "__init__.py" in _coconut.os.listdir(_coconut_dpath): + _coconut_paths_to_imp.append(_coconut_dpath) + for _coconut_imp_path in _coconut_paths_to_imp: + _coconut_imp_name = _coconut.os.path.splitext(_coconut.os.path.basename(_coconut_imp_path))[0] + descr = _coconut_imp.find_module(_coconut_imp_name, [_coconut.os.path.dirname(_coconut_imp_path)]) + try: + _coconut_imp.load_module(_coconut_imp_name, *descr) + except: + pass + _coconut_dirnames[:] = [] + """.strip()) if imp_all: - out += "\n" + """ - for _coconut_m in _coconut.tuple(_coconut_sys.modules.values()):{oind} - _coconut_d = _coconut.getattr(_coconut_m, "__dict__", None) - if _coconut_d is not None:{oind} - for _coconut_k, _coconut_v in _coconut_d.items():{oind} - if not _coconut_k.startswith("_"):{oind} - _coconut.locals()[_coconut_k] = _coconut_v{cind}{cind}{cind}{cind} - """.strip().format(oind=openindent, cind=closeindent) + out += "\n" + handle_indentation(""" +for _coconut_m in _coconut.tuple(_coconut_sys.modules.values()): + _coconut_d = _coconut.getattr(_coconut_m, "__dict__", None) + if _coconut_d is not None: + for _coconut_k, _coconut_v in _coconut_d.items(): + if not _coconut_k.startswith("_"): + _coconut.locals()[_coconut_k] = _coconut_v + """.strip()) else: - out += "\n" + """ - for _coconut_n, _coconut_m in _coconut.tuple(_coconut_sys.modules.items()):{oind} - _coconut.locals()[_coconut_n] = _coconut_m{cind} - """.strip().format(oind=openindent, cind=closeindent) + out += "\n" + handle_indentation(""" +for _coconut_n, _coconut_m in _coconut.tuple(_coconut_sys.modules.items()): + _coconut.locals()[_coconut_n] = _coconut_m + """.strip()) return out @@ -1344,14 +1345,15 @@ def match_data_handle(self, original, loc, tokens): arg_names = ", ".join(matcher.name_list) arg_tuple = arg_names + ("," if len(matcher.name_list) == 1 else "") - extra_stmts = '''def __new__(_cls, *{match_to_args_var}, **{match_to_kwargs_var}): - {oind}{match_check_var} = False - {matching} - {pattern_error} - return _coconut.tuple.__new__(_cls, ({arg_tuple})) - {cind}'''.format( - oind=openindent, - cind=closeindent, + extra_stmts = handle_indentation( + ''' +def __new__(_cls, *{match_to_args_var}, **{match_to_kwargs_var}): + {match_check_var} = False + {matching} + {pattern_error} + return _coconut.tuple.__new__(_cls, ({arg_tuple})) +'''.strip(), add_newline=True, + ).format( match_to_args_var=match_to_args_var, match_to_kwargs_var=match_to_kwargs_var, match_check_var=match_check_var, @@ -1426,29 +1428,30 @@ def data_handle(self, loc, tokens): if starred_arg is not None: attr_str += (" " if attr_str else "") + starred_arg if base_args: - extra_stmts += '''def __new__(_cls, {all_args}): - {oind}return _coconut.tuple.__new__(_cls, {base_args_tuple} + {starred_arg}) - {cind}@_coconut.classmethod - def _make(cls, iterable, {kwd_only}new=_coconut.tuple.__new__, len=_coconut.len): - {oind}result = new(cls, iterable) - if len(result) < {req_args}: - {oind}raise _coconut.TypeError("Expected at least {req_args} argument(s), got %d" % len(result)) - {cind}return result - {cind}def _asdict(self): - {oind}return _coconut.OrderedDict((f, _coconut.getattr(self, f)) for f in self._fields) - {cind}def __repr__(self): - {oind}return "{name}({args_for_repr})".format(**self._asdict()) - {cind}def _replace(_self, **kwds): - {oind}result = _self._make(_coconut.tuple(_coconut.map(kwds.pop, {quoted_base_args_tuple}, _self)) + kwds.pop("{starred_arg}", self.{starred_arg})) - if kwds: - {oind}raise _coconut.ValueError("Got unexpected field names: " + _coconut.repr(kwds.keys())) - {cind}return result - {cind}@_coconut.property - def {starred_arg}(self): - {oind}return self[{num_base_args}:] - {cind}'''.format( - oind=openindent, - cind=closeindent, + extra_stmts += handle_indentation( + ''' +def __new__(_cls, {all_args}): + return _coconut.tuple.__new__(_cls, {base_args_tuple} + {starred_arg}) +@_coconut.classmethod +def _make(cls, iterable, {kwd_only}new=_coconut.tuple.__new__, len=_coconut.len): + result = new(cls, iterable) + if len(result) < {req_args}: + raise _coconut.TypeError("Expected at least {req_args} argument(s), got %d" % len(result)) + return result +def _asdict(self): + return _coconut.OrderedDict((f, _coconut.getattr(self, f)) for f in self._fields) +def __repr__(self): + return "{name}({args_for_repr})".format(**self._asdict()) +def _replace(_self, **kwds): + result = _self._make(_coconut.tuple(_coconut.map(kwds.pop, {quoted_base_args_tuple}, _self)) + kwds.pop("{starred_arg}", self.{starred_arg})) + if kwds: + raise _coconut.ValueError("Got unexpected field names: " + _coconut.repr(kwds.keys())) + return result +@_coconut.property +def {starred_arg}(self): + return self[{num_base_args}:] + '''.strip(), add_newline=True, + ).format( name=name, args_for_repr=", ".join(arg + "={" + arg.lstrip("*") + "!r}" for arg in base_args + ["*" + starred_arg]), starred_arg=starred_arg, @@ -1460,36 +1463,38 @@ def {starred_arg}(self): kwd_only=("*, " if self.target.startswith("3") else ""), ) else: - extra_stmts += '''def __new__(_cls, *{arg}): - {oind}return _coconut.tuple.__new__(_cls, {arg}) - {cind}@_coconut.classmethod - def _make(cls, iterable, {kwd_only}new=_coconut.tuple.__new__, len=None): - {oind}return new(cls, iterable) - {cind}def _asdict(self): - {oind}return _coconut.OrderedDict([("{arg}", self[:])]) - {cind}def __repr__(self): - {oind}return "{name}(*{arg}=%r)" % (self[:],) - {cind}def _replace(_self, **kwds): - {oind}result = self._make(kwds.pop("{arg}", _self)) - if kwds: - {oind}raise _coconut.ValueError("Got unexpected field names: " + _coconut.repr(kwds.keys())) - {cind}return result - {cind}@_coconut.property - def {arg}(self): - {oind}return self[:] - {cind}'''.format( - oind=openindent, - cind=closeindent, + extra_stmts += handle_indentation( + ''' +def __new__(_cls, *{arg}): + return _coconut.tuple.__new__(_cls, {arg}) +@_coconut.classmethod +def _make(cls, iterable, {kwd_only}new=_coconut.tuple.__new__, len=None): + return new(cls, iterable) +def _asdict(self): + return _coconut.OrderedDict([("{arg}", self[:])]) +def __repr__(self): + return "{name}(*{arg}=%r)" % (self[:],) +def _replace(_self, **kwds): + result = self._make(kwds.pop("{arg}", _self)) + if kwds: + raise _coconut.ValueError("Got unexpected field names: " + _coconut.repr(kwds.keys())) + return result +@_coconut.property +def {arg}(self): + return self[:] + '''.strip(), add_newline=True, + ).format( name=name, arg=starred_arg, kwd_only=("*, " if self.target.startswith("3") else ""), ) elif saw_defaults: - extra_stmts += '''def __new__(_cls, {all_args}): - {oind}return _coconut.tuple.__new__(_cls, {args_tuple}) - {cind}'''.format( - oind=openindent, - cind=closeindent, + extra_stmts += handle_indentation( + ''' +def __new__(_cls, {all_args}): + return _coconut.tuple.__new__(_cls, {args_tuple}) + '''.strip(), add_newline=True, + ).format( all_args=", ".join(all_args), args_tuple="(" + ", ".join(base_args) + ("," if len(base_args) == 1 else "") + ")", ) @@ -1515,15 +1520,15 @@ def assemble_data(self, name, namedtuple_call, inherit, extra_stmts, stmts): ) # add universal statements - extra_stmts = '''__slots__ = () - __ne__ = _coconut.object.__ne__ - def __eq__(self, other): - {oind}return self.__class__ is other.__class__ and _coconut.tuple.__eq__(self, other) - {cind}def __hash__(self): - {oind}return _coconut.tuple.__hash__(self) ^ hash(self.__class__) - {cind}'''.format( - oind=openindent, - cind=closeindent, + extra_stmts = handle_indentation( + ''' +__slots__ = () +__ne__ = _coconut.object.__ne__ +def __eq__(self, other): + return self.__class__ is other.__class__ and _coconut.tuple.__eq__(self, other) +def __hash__(self): + return _coconut.tuple.__hash__(self) ^ hash(self.__class__) + '''.strip(), add_newline=True, ) + extra_stmts # manage docstring @@ -1598,16 +1603,24 @@ def pattern_error(self, original, loc, value_var, check_var, match_error_class=' base_line = clean(self.reformat(getline(loc, original))) line_wrap = self.wrap_str_of(base_line) repr_wrap = self.wrap_str_of(ascii(base_line)) - return ( - "if not " + check_var + ":\n" + openindent - + match_val_repr_var + " = _coconut.repr(" + value_var + ")\n" - + match_err_var + " = " + match_error_class + '("pattern-matching failed for " ' - + repr_wrap + ' " in " + (' + match_val_repr_var - + " if _coconut.len(" + match_val_repr_var + ") <= " + str(max_match_val_repr_len) - + " else " + match_val_repr_var + "[:" + str(max_match_val_repr_len) + '] + "..."))\n' - + match_err_var + ".pattern = " + line_wrap + "\n" - + match_err_var + ".value = " + value_var + "\n" - + "raise " + match_err_var + "\n" + closeindent + return handle_indentation( + """ +if not {check_var}: + {match_val_repr_var} = _coconut.repr({value_var}) + {match_err_var} = {match_error_class}("pattern-matching failed for " {repr_wrap} " in " + ({match_val_repr_var} if _coconut.len({match_val_repr_var}) <= {max_match_val_repr_len} else {match_val_repr_var}[:{max_match_val_repr_len}] + "...")) + {match_err_var}.pattern = {line_wrap} + {match_err_var}.value = {value_var} + raise {match_err_var} + """.strip(), add_newline=True, + ).format( + check_var=check_var, + match_val_repr_var=match_val_repr_var, + value_var=value_var, + match_err_var=match_err_var, + match_error_class=match_error_class, + repr_wrap=repr_wrap, + max_match_val_repr_len=max_match_val_repr_len, + line_wrap=line_wrap, ) def destructuring_stmt_handle(self, original, loc, tokens): diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index d09142000..51e11ccb0 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -50,6 +50,7 @@ use_computation_graph, py2_vers, py3_vers, + tabideal, ) from coconut.exceptions import ( CoconutException, @@ -601,3 +602,29 @@ def interleaved_join(outer_list, inner_list): interleaved.extend(xx) interleaved.append(outer_list[-1]) return "".join(interleaved) + + +def handle_indentation(inputstr, add_newline=False): + """Replace tabideal indentation with openindent and closeindent.""" + out_lines = [] + prev_ind = None + for line in inputstr.splitlines(): + new_ind_str, _ = split_leading_indent(line) + internal_assert(new_ind_str.strip(" ") == "", "invalid indentation characters for handle_indentation", new_ind_str) + internal_assert(len(new_ind_str) % tabideal == 0, "invalid indentation level for handle_indentation", len(new_ind_str)) + new_ind = len(new_ind_str) // tabideal + if prev_ind is None: # first line + indent = "" + elif new_ind > prev_ind: # indent + indent = openindent * (new_ind - prev_ind) + elif new_ind < prev_ind: # dedent + indent = closeindent * (prev_ind - new_ind) + else: + indent = "" + out_lines.append(indent + line) + prev_ind = new_ind + if add_newline: + out_lines.append("") + if prev_ind > 0: + out_lines[-1] += closeindent * prev_ind + return "\n".join(out_lines) From d024e5dbfeebd9267060d5ddd537e6c8a914add1 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 8 Jul 2019 19:29:10 -0700 Subject: [PATCH 142/163] Disable easter egg test on travis --- Makefile | 8 ++++---- tests/src/cocotest/agnostic/main.coco | 4 +--- tests/src/runner.coco | 3 ++- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index aed7fa2d6..c37fe0b3e 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ test-all: .PHONY: test-basic test-basic: python ./tests --strict --force - python ./tests/dest/runner.py + python ./tests/dest/runner.py --test-easter-egg python ./tests/dest/extras.py # same as test-basic, but doesn't recompile unchanged test files; @@ -31,21 +31,21 @@ test-basic: .PHONY: test-tests test-tests: python ./tests --strict - python ./tests/dest/runner.py + python ./tests/dest/runner.py --test-easter-egg python ./tests/dest/extras.py # same as test-basic but also runs mypy .PHONY: test-mypy test-mypy: python ./tests --strict --force --target sys --mypy --follow-imports silent --ignore-missing-imports - python ./tests/dest/runner.py + python ./tests/dest/runner.py --test-easter-egg python ./tests/dest/extras.py # same as test-basic but includes verbose output for better debugging .PHONY: test-verbose test-verbose: python ./tests --strict --force --verbose --jobs 0 - python ./tests/dest/runner.py + python ./tests/dest/runner.py --test-easter-egg python ./tests/dest/extras.py .PHONY: diff diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index d98ffc524..dc2a8defb 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -571,10 +571,8 @@ def easter_egg_test(): def tco_func() = tco_func() -def main(*args): +def main(test_easter_egg=False): """Asserts arguments and executes tests.""" - assert all(args) - print(".", end="") # .. assert main_test() diff --git a/tests/src/runner.coco b/tests/src/runner.coco index 2c7246e22..8de665502 100644 --- a/tests/src/runner.coco +++ b/tests/src/runner.coco @@ -7,4 +7,5 @@ from cocotest.main import main if __name__ == "__main__": print(".", end="") # . - main(cocotest.__doc__) + assert cocotest.__doc__ + main(test_easter_egg="--test-easter-egg" in sys.argv) From 8305ad851ac8186aa98acda2e997350ff0acfeb2 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 9 Jul 2019 16:21:33 -0700 Subject: [PATCH 143/163] Fix easter egg testing --- coconut/compiler/compiler.py | 2 +- tests/src/cocotest/agnostic/main.coco | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index adbe701b9..af820a3aa 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -1568,9 +1568,9 @@ def import_handle(self, original, loc, tokens): raise CoconutInternalException("invalid import tokens", tokens) imports = list(imports) if imp_from == "*" or imp_from is None and "*" in imports: - logger.warn_err(self.make_err(CoconutSyntaxWarning, "[from *] import * is a Coconut Easter egg and should not be used in production code", original, loc)) if not (len(imports) == 1 and imports[0] == "*"): raise self.make_err(CoconutSyntaxError, "only [from *] import * allowed, not from * import name", original, loc) + logger.warn_err(self.make_err(CoconutSyntaxWarning, "[from *] import * is a Coconut Easter egg and should not be used in production code", original, loc)) return special_starred_import_handle(imp_all=bool(imp_from)) if self.strict: self.unused_imports.update(imported_names(imports)) diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index dc2a8defb..51cee7018 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -614,7 +614,8 @@ def main(test_easter_egg=False): print(".", end="") # ........ from . import tutorial # type: ignore - print(".", end="") # ......... - assert easter_egg_test() + if test_easter_egg: + print(".", end="") # ......... + assert easter_egg_test() print("\n") From b5f893a5fb39d87702e6f959825cbf6218abc152 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 9 Jul 2019 17:14:30 -0700 Subject: [PATCH 144/163] Fix failing tests --- coconut/stubs/coconut/convenience.pyi | 7 +++++++ tests/constants_test.py | 8 ++++++++ tests/main_test.py | 9 +++++++++ 3 files changed, 24 insertions(+) diff --git a/coconut/stubs/coconut/convenience.pyi b/coconut/stubs/coconut/convenience.pyi index 6c6c4df82..a2c768422 100644 --- a/coconut/stubs/coconut/convenience.pyi +++ b/coconut/stubs/coconut/convenience.pyi @@ -58,6 +58,13 @@ PARSERS: Dict[Text, Callable] = ... def parse(code: Text, mode: Text) -> Text: ... +def coconut_eval( + expression: Text, + globals: Dict[Text, Any]=None, + locals: Dict[Text, Any]=None, +) -> Any: ... + + # ----------------------------------------------------------------------------------------------------------------------- # IMPORTER: # ----------------------------------------------------------------------------------------------------------------------- diff --git a/tests/constants_test.py b/tests/constants_test.py index 6ab95df76..a50db34dc 100644 --- a/tests/constants_test.py +++ b/tests/constants_test.py @@ -97,3 +97,11 @@ def test_imports(self): assert is_importable(new_imp), "Failed to import " + new_imp else: assert is_importable(old_imp), "Failed to import " + old_imp + + +# ----------------------------------------------------------------------------------------------------------------------- +# MAIN: +# ----------------------------------------------------------------------------------------------------------------------- + +if __name__ == "__main__": + unittest.main() diff --git a/tests/main_test.py b/tests/main_test.py index cddd6b964..69ff4b56d 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -412,6 +412,7 @@ def test_jupyter_console(self): p = pexpect.spawn(cmd) p.expect("In", timeout=100) p.sendeof() + p.expect("Do you really want to exit") p.sendline("y") p.expect("Shutting down kernel|shutting down|Jupyter error") if p.isalive(): @@ -487,3 +488,11 @@ def test_pyston(self): comp_pyston(["--no-tco"]) if PY2 and PYPY: run_pyston() + + +# ----------------------------------------------------------------------------------------------------------------------- +# MAIN: +# ----------------------------------------------------------------------------------------------------------------------- + +if __name__ == "__main__": + unittest.main() From 12e81e7a65610c2de7530ad72374adb37579162c Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 9 Jul 2019 20:22:17 -0700 Subject: [PATCH 145/163] Fix prelude test --- tests/main_test.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/tests/main_test.py b/tests/main_test.py index 69ff4b56d..94be8abca 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -162,17 +162,24 @@ def comp(path=None, folder=None, file=None, args=[], **kwargs): call_coconut([source, compdest] + args, **kwargs) +def rm_path(path): + """Delete a path.""" + if os.path.isdir(path): + shutil.rmtree(path) + elif os.path.isfile(path): + os.remove(path) + + @contextmanager -def remove_when_done(path): - """Removes a path when done.""" +def using_path(path): + """Removes a path at the beginning and end.""" + if os.path.exists(path): + rm_path(path) try: yield finally: try: - if os.path.isdir(path): - shutil.rmtree(path) - elif os.path.isfile(path): - os.remove(path) + rm_path(path) except OSError: logger.display_exc() @@ -314,9 +321,9 @@ def run_pyprover(**kwargs): def comp_prelude(args=[], **kwargs): """Compiles evhub/coconut-prelude.""" call(["git", "clone", prelude_git]) - call_coconut([os.path.join(prelude, "setup.coco"), "--strict"] + args, **kwargs) if PY36: args.extend(["--target", "3.6", "--mypy"]) + call_coconut([os.path.join(prelude, "setup.coco"), "--strict"] + args, **kwargs) call_coconut([os.path.join(prelude, "prelude-source"), os.path.join(prelude, "prelude"), "--strict"] + args, **kwargs) @@ -366,7 +373,7 @@ def test_import_hook(self): sys.path.append(src) auto_compilation(True) try: - with remove_when_done(runnable_py): + with using_path(runnable_py): with using_logger(): import runnable reload(runnable) @@ -376,14 +383,14 @@ def test_import_hook(self): assert runnable.success == "" def test_runnable(self): - with remove_when_done(runnable_py): + with using_path(runnable_py): run_runnable() def test_runnable_nowrite(self): run_runnable(["-n"]) def test_compile_to_file(self): - with remove_when_done(runnable_py): + with using_path(runnable_py): call_coconut([runnable_coco, runnable_py]) call_python([runnable_py, "--arg"], assert_output=True) @@ -472,19 +479,19 @@ def test_simple_minify(self): class TestExternal(unittest.TestCase): def test_pyprover(self): - with remove_when_done(pyprover): + with using_path(pyprover): comp_pyprover() run_pyprover() if not PYPY or PY2: def test_prelude(self): - with remove_when_done(prelude): + with using_path(prelude): comp_prelude() if PY35: # has typing run_prelude() def test_pyston(self): - with remove_when_done(pyston): + with using_path(pyston): comp_pyston(["--no-tco"]) if PY2 and PYPY: run_pyston() From d3831c8d3b67ef354e996edf8387c400dd8985f6 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 10 Jul 2019 12:52:49 -0700 Subject: [PATCH 146/163] Fix tests --- tests/main_test.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/main_test.py b/tests/main_test.py index 94be8abca..160d1a8f8 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -192,8 +192,13 @@ def using_dest(): except Exception: shutil.rmtree(dest) os.mkdir(dest) - with remove_when_done(dest): + try: yield + finally: + try: + rm_path(path) + except OSError: + logger.display_exc() @contextmanager From 6798ffccae1318477c4ef9f7f0edf1dabd07f902 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 10 Jul 2019 13:14:44 -0700 Subject: [PATCH 147/163] Fix typo --- tests/main_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/main_test.py b/tests/main_test.py index 160d1a8f8..3ae962602 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -196,7 +196,7 @@ def using_dest(): yield finally: try: - rm_path(path) + rm_path(dest) except OSError: logger.display_exc() From 7d2c24b8c0b7ddba5c23d21e9637834fc10e5156 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 11 Jul 2019 11:27:06 -0700 Subject: [PATCH 148/163] Fix prelude test --- tests/main_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/main_test.py b/tests/main_test.py index 3ae962602..288fcc5af 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -335,7 +335,7 @@ def comp_prelude(args=[], **kwargs): def run_prelude(**kwargs): """Runs coconut-prelude.""" call(["pip", "install", "-e", prelude]) - call_python([os.path.join(prelude, "prelude", "prelude_test.py")], assert_output=True, **kwargs) + call(["pytest", "--strict", "-s", os.path.join(prelude, "prelude")], assert_output=True, **kwargs) def comp_all(args=[], **kwargs): From 57d2fe90b536511ecc1646d0f840a4823b600f7a Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 12 Jul 2019 11:26:23 -0700 Subject: [PATCH 149/163] Further fix prelude test --- tests/main_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/main_test.py b/tests/main_test.py index 288fcc5af..e711a95a4 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -335,7 +335,7 @@ def comp_prelude(args=[], **kwargs): def run_prelude(**kwargs): """Runs coconut-prelude.""" call(["pip", "install", "-e", prelude]) - call(["pytest", "--strict", "-s", os.path.join(prelude, "prelude")], assert_output=True, **kwargs) + call(["pytest", "--strict", "-s", os.path.join(prelude, "prelude")], assert_output="passed", **kwargs) def comp_all(args=[], **kwargs): From 40d60ec9e420746af98723c66cf139f9490911e0 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 17 Jul 2019 22:19:41 -0700 Subject: [PATCH 150/163] Bump dependencies --- coconut/constants.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index 3a73a1201..e3704b974 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -176,12 +176,12 @@ def checksum(data): "pyparsing": (2, 4, 0), "cPyparsing": (2, 4, 0, 1, 0, 0), "pre-commit": (1,), - "pygments": (2, 3), + "pygments": (2, 4), "recommonmark": (0, 5), "psutil": (5,), "jupyter": (1, 0), - "mypy": (0, 701), - "futures": (3, 2), + "mypy": (0, 720), + "futures": (3, 3), "backports.functools-lru-cache": (1, 5), "argparse": (1, 4), "pexpect": (4,), @@ -189,10 +189,11 @@ def checksum(data): "trollius": (2, 2), "requests": (2,), "numpy": (1,), - "prompt_toolkit:3": (1,), - ("ipython", "py3"): (7, 3), + ("ipython", "py3"): (7, 6), ("jupyter-console", "py3"): (6,), ("ipykernel", "py3"): (5, 1), + # don't upgrade this to allow all versions + "prompt_toolkit:3": (1,), # don't upgrade this; it breaks on Python 2.6 "pytest": (3,), # don't upgrade this; it breaks on unix From ff455e27ce4dc886e602db6a3c2ea93cb7f9d66e Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 Jul 2019 14:55:55 -0700 Subject: [PATCH 151/163] Fix subscript tuples Resolves #511. --- Makefile | 15 +++++++++++---- coconut/compiler/grammar.py | 2 +- coconut/root.py | 2 +- tests/src/cocotest/agnostic/main.coco | 4 ++-- tests/src/cocotest/agnostic/suite.coco | 2 ++ tests/src/runner.coco | 2 +- 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index c37fe0b3e..f4fc775a7 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ test-all: .PHONY: test-basic test-basic: python ./tests --strict --force - python ./tests/dest/runner.py --test-easter-egg + python ./tests/dest/runner.py python ./tests/dest/extras.py # same as test-basic, but doesn't recompile unchanged test files; @@ -31,21 +31,28 @@ test-basic: .PHONY: test-tests test-tests: python ./tests --strict - python ./tests/dest/runner.py --test-easter-egg + python ./tests/dest/runner.py python ./tests/dest/extras.py # same as test-basic but also runs mypy .PHONY: test-mypy test-mypy: python ./tests --strict --force --target sys --mypy --follow-imports silent --ignore-missing-imports - python ./tests/dest/runner.py --test-easter-egg + python ./tests/dest/runner.py python ./tests/dest/extras.py # same as test-basic but includes verbose output for better debugging .PHONY: test-verbose test-verbose: python ./tests --strict --force --verbose --jobs 0 - python ./tests/dest/runner.py --test-easter-egg + python ./tests/dest/runner.py + python ./tests/dest/extras.py + +# same as test-basic but also tests easter eggs +.PHONY: test-easter-eggs +test-easter-eggs: + python ./tests --strict --force + python ./tests/dest/runner.py --test-easter-eggs python ./tests/dest/extras.py .PHONY: diff diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index afeef484f..33ec0c879 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -1043,7 +1043,7 @@ class Grammar(object): slicetest = Optional(test_no_chain) sliceop = condense(unsafe_colon + slicetest) subscript = condense(slicetest + sliceop + Optional(sliceop)) | test - subscriptlist = itemlist(subscript, comma) + subscriptlist = itemlist(subscript, comma, suppress_trailing=False) slicetestgroup = Optional(test_no_chain, default="") sliceopgroup = unsafe_colon.suppress() + slicetestgroup diff --git a/coconut/root.py b/coconut/root.py index a1934f975..3360b4971 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "1.4.0" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 42 +DEVELOP = 43 # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: diff --git a/tests/src/cocotest/agnostic/main.coco b/tests/src/cocotest/agnostic/main.coco index 51cee7018..338b3e90a 100644 --- a/tests/src/cocotest/agnostic/main.coco +++ b/tests/src/cocotest/agnostic/main.coco @@ -571,7 +571,7 @@ def easter_egg_test(): def tco_func() = tco_func() -def main(test_easter_egg=False): +def main(test_easter_eggs=False): """Asserts arguments and executes tests.""" print(".", end="") # .. assert main_test() @@ -614,7 +614,7 @@ def main(test_easter_egg=False): print(".", end="") # ........ from . import tutorial # type: ignore - if test_easter_egg: + if test_easter_eggs: print(".", end="") # ......... assert easter_egg_test() diff --git a/tests/src/cocotest/agnostic/suite.coco b/tests/src/cocotest/agnostic/suite.coco index a0b61ee42..422ff76a2 100644 --- a/tests/src/cocotest/agnostic/suite.coco +++ b/tests/src/cocotest/agnostic/suite.coco @@ -330,6 +330,8 @@ def suite_test(): assert identity.method(*(1,), **{"a": 2}) == ((1,), {"a": 2}) assert (.method(*(1,), **{"a": 2}))(identity) == ((1,), {"a": 2}) assert identity |> .method(*(1,), **{"a": 2}) == ((1,), {"a": 2}) + assert identity[1] == 1 + assert identity[1,] == (1,) assert container(1) == container(1) assert not container(1) != container(1) assert container(1) != container(2) diff --git a/tests/src/runner.coco b/tests/src/runner.coco index 8de665502..e5562672f 100644 --- a/tests/src/runner.coco +++ b/tests/src/runner.coco @@ -8,4 +8,4 @@ from cocotest.main import main if __name__ == "__main__": print(".", end="") # . assert cocotest.__doc__ - main(test_easter_egg="--test-easter-egg" in sys.argv) + main(test_easter_eggs="--test-easter-eggs" in sys.argv) From 00b846b9fd9f8dd7723431c91609b898a55dd25d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 Jul 2019 15:08:21 -0700 Subject: [PATCH 152/163] Improve formatting --- .pre-commit-config.yaml | 8 +- CONTRIBUTING.md | 3 +- coconut/command/command.py | 6 +- coconut/command/util.py | 8 +- coconut/compiler/compiler.py | 80 +++++---- coconut/compiler/grammar.py | 320 ++++++++++++++++++++--------------- coconut/compiler/matching.py | 6 +- coconut/compiler/util.py | 12 +- setup.py | 10 +- 9 files changed, 268 insertions(+), 185 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f0d16ab85..f3e9a343d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: git://github.com/pre-commit/pre-commit-hooks.git - rev: v2.1.0 + rev: v2.2.3 hooks: - id: check-byte-order-marker - id: check-merge-conflict @@ -17,9 +17,9 @@ repos: - --autofix - id: flake8 args: - - --ignore=W503,E501,E265,E402,F405,E305 + - --ignore=W503,E501,E265,E402,F405,E305,E126 - repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v1.4.3 + rev: v1.4.4 hooks: - id: autopep8 args: @@ -29,6 +29,6 @@ repos: - --experimental - --ignore=W503,E501,E722,E402 - repo: https://github.com/asottile/add-trailing-comma - rev: v0.7.2 + rev: v1.4.1 hooks: - id: add-trailing-comma diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a8c6b7929..e7c878f3c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -161,12 +161,13 @@ After you've tested your changes locally, you'll want to add more permanent test 1. Preparation: 1. Run `make check-reqs` and update dependencies as necessary 1. Run `make format` - 1. Check changes in [`compiled-cocotest`](https://github.com/evhub/compiled-cocotest) and [`pyprover`](https://github.com/evhub/pyprover) + 1. Check changes in [`compiled-cocotest`](https://github.com/evhub/compiled-cocotest), [`pyprover`](https://github.com/evhub/pyprover), and [`coconut-prelude`](https://github.com/evhub/coconut-prelude) 1. Check [Codacy issues](https://www.codacy.com/app/evanjhub) (for `coconut` and `compiled-cocotest`) and [LGTM alerts](https://lgtm.com/projects/g/evhub/coconut/) 1. Make sure [`coconut-develop`](https://pypi.python.org/pypi/coconut-develop) package looks good 1. Run `make docs` and ensure local documentation looks good 1. Make sure [develop documentation](http://coconut.readthedocs.io/en/develop/) looks good 1. Make sure [Travis](https://travis-ci.org/evhub/coconut/builds) and [AppVeyor](https://ci.appveyor.com/project/evhub/coconut) are passing + 1. Run `make test-easter-eggs` 1. Turn off `develop` in `root.py` 1. Set `root.py` to new version number 1. If major release, set `root.py` to new version name diff --git a/coconut/command/command.py b/coconut/command/command.py index b8d74208a..7c39d4fe4 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -255,7 +255,8 @@ def use_args(self, args, interact=True, original_args=None): logger.log("Reading piped input from stdin...") self.execute(self.comp.parse_block(sys.stdin.read())) got_stdin = True - if args.interact or (interact and not ( + if args.interact or ( + interact and not ( got_stdin or args.source or args.code @@ -263,7 +264,8 @@ def use_args(self, args, interact=True, original_args=None): or args.documentation or args.watch or args.jupyter is not None - )): + ) + ): self.start_prompt() if args.watch: self.watch(source, dest, package, args.run, args.force) diff --git a/coconut/command/util.py b/coconut/command/util.py index 51c1a2683..37545be11 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -93,9 +93,11 @@ except ImportError: prompt_toolkit = None except KeyError: - complain(ImportError( - "detected outdated pygments version (run 'pip install --upgrade pygments' to fix)", - )) + complain( + ImportError( + "detected outdated pygments version (run 'pip install --upgrade pygments' to fix)", + ), + ) prompt_toolkit = None # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index af820a3aa..03d085e8c 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -258,7 +258,8 @@ def imported_names(imports): def special_starred_import_handle(imp_all=False): """Handles the [from *] import * Coconut Easter egg.""" - out = handle_indentation(""" + out = handle_indentation( + """ import imp as _coconut_imp for _coconut_base_path in _coconut_sys.path: for _coconut_dirpath, _coconut_dirnames, _coconut_filenames in _coconut.os.walk(_coconut_base_path): @@ -279,21 +280,26 @@ def special_starred_import_handle(imp_all=False): except: pass _coconut_dirnames[:] = [] - """.strip()) + """.strip() + ) if imp_all: - out += "\n" + handle_indentation(""" + out += "\n" + handle_indentation( + """ for _coconut_m in _coconut.tuple(_coconut_sys.modules.values()): _coconut_d = _coconut.getattr(_coconut_m, "__dict__", None) if _coconut_d is not None: for _coconut_k, _coconut_v in _coconut_d.items(): if not _coconut_k.startswith("_"): _coconut.locals()[_coconut_k] = _coconut_v - """.strip()) + """.strip() + ) else: - out += "\n" + handle_indentation(""" + out += "\n" + handle_indentation( + """ for _coconut_n, _coconut_m in _coconut.tuple(_coconut_sys.modules.items()): _coconut.locals()[_coconut_n] = _coconut_m - """.strip()) + """.strip() + ) return out @@ -443,15 +449,17 @@ def genhash(self, code, package_level=-1): "package_level": package_level, }, ) - return hex(checksum( - hash_sep.join( - str(item) for item in ( - (VERSION_STR,) - + reduce_args - + (package_level, code) - ) - ).encode(default_encoding), - )) + return hex( + checksum( + hash_sep.join( + str(item) for item in ( + (VERSION_STR,) + + reduce_args + + (package_level, code) + ) + ).encode(default_encoding), + ), + ) def reset(self): """Resets references.""" @@ -517,14 +525,18 @@ def bind(self): self.case_stmt <<= trace(attach(self.case_stmt_ref, self.case_stmt_handle)) self.f_string <<= attach(self.f_string_ref, self.f_string_handle) - self.decoratable_normal_funcdef_stmt <<= trace(attach( - self.decoratable_normal_funcdef_stmt_ref, - self.decoratable_funcdef_stmt_handle, - )) - self.decoratable_async_funcdef_stmt <<= trace(attach( - self.decoratable_async_funcdef_stmt_ref, - partial(self.decoratable_funcdef_stmt_handle, is_async=True), - )) + self.decoratable_normal_funcdef_stmt <<= trace( + attach( + self.decoratable_normal_funcdef_stmt_ref, + self.decoratable_funcdef_stmt_handle, + ), + ) + self.decoratable_async_funcdef_stmt <<= trace( + attach( + self.decoratable_async_funcdef_stmt_ref, + partial(self.decoratable_funcdef_stmt_handle, is_async=True), + ), + ) self.u_string <<= attach(self.u_string_ref, self.u_string_check) self.matrix_at <<= attach(self.matrix_at_ref, self.matrix_at_check) @@ -1053,10 +1065,12 @@ def ln_comment(self, ln): # CoconutInternalExceptions should always be caught and complained here if self.keep_lines: if not 1 <= ln <= len(self.original_lines) + 2: - complain(CoconutInternalException( - "out of bounds line number", ln, - "not in range [1, " + str(len(self.original_lines) + 2) + "]", - )) + complain( + CoconutInternalException( + "out of bounds line number", ln, + "not in range [1, " + str(len(self.original_lines) + 2) + "]", + ), + ) if ln >= len(self.original_lines) + 1: # trim too large lni = -1 else: @@ -1352,7 +1366,7 @@ def __new__(_cls, *{match_to_args_var}, **{match_to_kwargs_var}): {matching} {pattern_error} return _coconut.tuple.__new__(_cls, ({arg_tuple})) -'''.strip(), add_newline=True, + '''.strip(), add_newline=True, ).format( match_to_args_var=match_to_args_var, match_to_kwargs_var=match_to_kwargs_var, @@ -1450,7 +1464,7 @@ def _replace(_self, **kwds): @_coconut.property def {starred_arg}(self): return self[{num_base_args}:] - '''.strip(), add_newline=True, + '''.strip(), add_newline=True, ).format( name=name, args_for_repr=", ".join(arg + "={" + arg.lstrip("*") + "!r}" for arg in base_args + ["*" + starred_arg]), @@ -1482,7 +1496,7 @@ def _replace(_self, **kwds): @_coconut.property def {arg}(self): return self[:] - '''.strip(), add_newline=True, + '''.strip(), add_newline=True, ).format( name=name, arg=starred_arg, @@ -1493,7 +1507,7 @@ def {arg}(self): ''' def __new__(_cls, {all_args}): return _coconut.tuple.__new__(_cls, {args_tuple}) - '''.strip(), add_newline=True, + '''.strip(), add_newline=True, ).format( all_args=", ".join(all_args), args_tuple="(" + ", ".join(base_args) + ("," if len(base_args) == 1 else "") + ")", @@ -1528,7 +1542,7 @@ def __eq__(self, other): return self.__class__ is other.__class__ and _coconut.tuple.__eq__(self, other) def __hash__(self): return _coconut.tuple.__hash__(self) ^ hash(self.__class__) - '''.strip(), add_newline=True, + '''.strip(), add_newline=True, ) + extra_stmts # manage docstring @@ -1611,7 +1625,7 @@ def pattern_error(self, original, loc, value_var, check_var, match_error_class=' {match_err_var}.pattern = {line_wrap} {match_err_var}.value = {value_var} raise {match_err_var} - """.strip(), add_newline=True, + """.strip(), add_newline=True, ).format( check_var=check_var, match_val_repr_var=match_val_repr_var, diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 33ec0c879..f0763cd5d 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -901,10 +901,12 @@ class Grammar(object): dict_comp_ref = lbrace.suppress() + (test + colon.suppress() + test | dubstar_expr) + comp_for + rbrace.suppress() dict_item = condense( lbrace - + Optional(itemlist( - addspace(condense(test + colon) + test) | dubstar_expr, - comma, - )) + + Optional( + itemlist( + addspace(condense(test + colon) + test) | dubstar_expr, + comma, + ), + ) + rbrace, ) test_expr = yield_expr | testlist_star_expr @@ -989,30 +991,48 @@ class Grammar(object): just_op = just_star | just_slash match = Forward() - args_list = ~just_op + trace(addspace(ZeroOrMore(condense( - # everything here must end with arg_comma - (star | dubstar) + tfpdef - | star_sep_arg - | slash_sep_arg - | tfpdef_default, - )))) + args_list = ~just_op + trace( + addspace( + ZeroOrMore( + condense( + # everything here must end with arg_comma + (star | dubstar) + tfpdef + | star_sep_arg + | slash_sep_arg + | tfpdef_default, + ), + ), + ), + ) parameters = condense(lparen + args_list + rparen) - var_args_list = ~just_op + trace(addspace(ZeroOrMore(condense( - # everything here must end with vararg_comma - (star | dubstar) + name + vararg_comma - | star_sep_vararg - | slash_sep_vararg - | name + Optional(default) + vararg_comma, - )))) - match_args_list = trace(Group(Optional(tokenlist( + var_args_list = ~just_op + trace( + addspace( + ZeroOrMore( + condense( + # everything here must end with vararg_comma + (star | dubstar) + name + vararg_comma + | star_sep_vararg + | slash_sep_vararg + | name + Optional(default) + vararg_comma, + ), + ), + ), + ) + match_args_list = trace( Group( - (star | dubstar) + match - | star # not star_sep because pattern-matching can handle star separators on any Python version - | slash # not slash_sep as above - | match + Optional(equals.suppress() + test), + Optional( + tokenlist( + Group( + (star | dubstar) + match + | star # not star_sep because pattern-matching can handle star separators on any Python version + | slash # not slash_sep as above + | match + Optional(equals.suppress() + test), + ), + comma, + ), + ), ), - comma, - )))) + ) call_item = ( dubstar + test @@ -1028,13 +1048,15 @@ class Grammar(object): | tokenlist(Group(call_item), comma) + rparen.suppress() ) function_call = attach(function_call_tokens, function_call_handle) - questionmark_call_tokens = Group(tokenlist( - Group( - questionmark - | call_item, + questionmark_call_tokens = Group( + tokenlist( + Group( + questionmark + | call_item, + ), + comma, ), - comma, - )) + ) methodcaller_args = ( itemlist(condense(call_item), comma) | op_item @@ -1361,9 +1383,11 @@ class Grammar(object): import_names = Group(maybeparens(lparen, tokenlist(dotted_as_name, comma), rparen)) from_import_names = Group(maybeparens(lparen, tokenlist(import_as_name, comma), rparen)) basic_import = keyword("import").suppress() - (import_names | Group(star)) - from_import = (keyword("from").suppress() - - condense(ZeroOrMore(unsafe_dot) + dotted_name | OneOrMore(unsafe_dot) | star) - - keyword("import").suppress() - (from_import_names | Group(star))) + from_import = ( + keyword("from").suppress() + - condense(ZeroOrMore(unsafe_dot) + dotted_name | OneOrMore(unsafe_dot) | star) + - keyword("import").suppress() - (from_import_names | Group(star)) + ) import_stmt = Forward() import_stmt_ref = from_import | basic_import @@ -1421,18 +1445,20 @@ class Grammar(object): lbrack.suppress() + matchlist_star + rbrack.suppress() | lparen.suppress() + matchlist_star + rparen.suppress() )("star") - base_match = trace(Group( - match_string - | match_const("const") - | (lparen.suppress() + match + rparen.suppress())("paren") - | (lbrace.suppress() + matchlist_dict + Optional(dubstar.suppress() + name) + rbrace.suppress())("dict") - | (Optional(set_s.suppress()) + lbrace.suppress() + matchlist_set + rbrace.suppress())("set") - | iter_match - | series_match - | star_match - | (name + lparen.suppress() + matchlist_data + rparen.suppress())("data") - | name("var"), - )) + base_match = trace( + Group( + match_string + | match_const("const") + | (lparen.suppress() + match + rparen.suppress())("paren") + | (lbrace.suppress() + matchlist_dict + Optional(dubstar.suppress() + name) + rbrace.suppress())("dict") + | (Optional(set_s.suppress()) + lbrace.suppress() + matchlist_set + rbrace.suppress())("set") + | iter_match + | series_match + | star_match + | (name + lparen.suppress() + matchlist_data + rparen.suppress())("data") + | name("var"), + ), + ) matchlist_trailer = base_match + OneOrMore(keyword("as") + name | keyword("is") + atom_item) as_match = Group(matchlist_trailer("trailer")) | base_match matchlist_and = as_match + OneOrMore(keyword("and").suppress() + as_match) @@ -1443,19 +1469,23 @@ class Grammar(object): else_stmt = condense(keyword("else") - suite) full_suite = colon.suppress() + Group((newline.suppress() + indent.suppress() + OneOrMore(stmt) + dedent.suppress()) | simple_stmt) - full_match = trace(attach( - keyword("match").suppress() + match + addspace(Optional(keyword("not")) + keyword("in")) - test - match_guard - full_suite, - match_handle, - )) + full_match = trace( + attach( + keyword("match").suppress() + match + addspace(Optional(keyword("not")) + keyword("in")) - test - match_guard - full_suite, + match_handle, + ), + ) match_stmt = condense(full_match - Optional(else_stmt)) destructuring_stmt = Forward() destructuring_stmt_ref = Optional(keyword("match").suppress()) + match + equals.suppress() + test_expr case_stmt = Forward() - case_match = trace(Group( - keyword("match").suppress() - match - Optional(keyword("if").suppress() - test) - full_suite, - )) + case_match = trace( + Group( + keyword("match").suppress() - match - Optional(keyword("if").suppress() - test) - full_suite, + ), + ) case_stmt_ref = ( keyword("case").suppress() + test - colon.suppress() - newline.suppress() - indent.suppress() - Group(OneOrMore(case_match)) @@ -1477,13 +1507,15 @@ class Grammar(object): ) - Optional(keyword("as").suppress() - name), except_handle, ) - try_stmt = condense(keyword("try") - suite + ( - keyword("finally") - suite - | ( - OneOrMore(except_clause - suite) - Optional(keyword("except") - suite) - | keyword("except") - suite - ) - Optional(else_stmt) - Optional(keyword("finally") - suite) - )) + try_stmt = condense( + keyword("try") - suite + ( + keyword("finally") - suite + | ( + OneOrMore(except_clause - suite) - Optional(keyword("except") - suite) + | keyword("except") - suite + ) - Optional(else_stmt) - Optional(keyword("finally") - suite) + ), + ) exec_stmt_ref = keyword("exec").suppress() + lparen.suppress() + test + Optional( comma.suppress() + test + Optional( comma.suppress() + test + Optional( @@ -1517,31 +1549,37 @@ class Grammar(object): name_match_funcdef = Forward() op_match_funcdef = Forward() - op_match_funcdef_arg = Group(Optional( - lparen.suppress() - + Group(match + Optional(equals.suppress() + test)) - + rparen.suppress(), - )) + op_match_funcdef_arg = Group( + Optional( + lparen.suppress() + + Group(match + Optional(equals.suppress() + test)) + + rparen.suppress(), + ), + ) name_match_funcdef_ref = keyword("def").suppress() + dotted_name + lparen.suppress() + match_args_list + match_guard + rparen.suppress() op_match_funcdef_ref = keyword("def").suppress() + op_match_funcdef_arg + op_funcdef_name + op_match_funcdef_arg + match_guard base_match_funcdef = trace(op_match_funcdef | name_match_funcdef) - def_match_funcdef = trace(attach( - base_match_funcdef - + colon.suppress() - + ( - attach(simple_stmt, make_suite_handle) - | newline.suppress() + indent.suppress() - + Optional(docstring) - + attach(condense(OneOrMore(stmt)), make_suite_handle) - + dedent.suppress() + def_match_funcdef = trace( + attach( + base_match_funcdef + + colon.suppress() + + ( + attach(simple_stmt, make_suite_handle) + | newline.suppress() + indent.suppress() + + Optional(docstring) + + attach(condense(OneOrMore(stmt)), make_suite_handle) + + dedent.suppress() + ), + join_match_funcdef, ), - join_match_funcdef, - )) - match_def_modifiers = trace(Optional( - # we don't suppress addpattern so its presence can be detected later - keyword("match").suppress() + Optional(keyword("addpattern")) - | keyword("addpattern") + Optional(keyword("match")).suppress(), - )) + ) + match_def_modifiers = trace( + Optional( + # we don't suppress addpattern so its presence can be detected later + keyword("match").suppress() + Optional(keyword("addpattern")) + | keyword("addpattern") + Optional(keyword("match")).suppress(), + ), + ) match_funcdef = addspace(match_def_modifiers + def_match_funcdef) where_suite = colon.suppress() - Group( @@ -1564,56 +1602,68 @@ class Grammar(object): | condense(newline - indent - math_funcdef_body - dedent) ) end_func_equals = return_typedef + equals.suppress() | fixto(equals, ":") - math_funcdef = trace(attach( - condense(addspace(keyword("def") + base_funcdef) + end_func_equals) - math_funcdef_suite, - math_funcdef_handle, - )) + math_funcdef = trace( + attach( + condense(addspace(keyword("def") + base_funcdef) + end_func_equals) - math_funcdef_suite, + math_funcdef_handle, + ), + ) math_match_funcdef = addspace( match_def_modifiers - + trace(attach( - base_match_funcdef - + equals.suppress() - - Optional(docstring) - - ( - attach(implicit_return_stmt, make_suite_handle) - | newline.suppress() - indent.suppress() + + trace( + attach( + base_match_funcdef + + equals.suppress() - Optional(docstring) - - attach(math_funcdef_body, make_suite_handle) - - dedent.suppress() + - ( + attach(implicit_return_stmt, make_suite_handle) + | newline.suppress() - indent.suppress() + - Optional(docstring) + - attach(math_funcdef_body, make_suite_handle) + - dedent.suppress() + ), + join_match_funcdef, ), - join_match_funcdef, - )), + ), ) async_stmt = Forward() async_stmt_ref = addspace(keyword("async") + (with_stmt | for_stmt)) async_funcdef = keyword("async").suppress() + (funcdef | math_funcdef) - async_match_funcdef = addspace(trace( - # we don't suppress addpattern so its presence can be detected later - keyword("match").suppress() + keyword("addpattern") + keyword("async").suppress() - | keyword("addpattern") + keyword("match").suppress() + keyword("async").suppress() - | keyword("match").suppress() + keyword("async").suppress() + Optional(keyword("addpattern")) - | keyword("addpattern") + keyword("async").suppress() + Optional(keyword("match")).suppress() - | keyword("async").suppress() + match_def_modifiers, - ) + (def_match_funcdef | math_match_funcdef)) + async_match_funcdef = addspace( + trace( + # we don't suppress addpattern so its presence can be detected later + keyword("match").suppress() + keyword("addpattern") + keyword("async").suppress() + | keyword("addpattern") + keyword("match").suppress() + keyword("async").suppress() + | keyword("match").suppress() + keyword("async").suppress() + Optional(keyword("addpattern")) + | keyword("addpattern") + keyword("async").suppress() + Optional(keyword("match")).suppress() + | keyword("async").suppress() + match_def_modifiers, + ) + (def_match_funcdef | math_match_funcdef), + ) datadef = Forward() - data_args = Group(Optional(lparen.suppress() + ZeroOrMore( - Group( - # everything here must end with arg_comma - (name + arg_comma.suppress())("name") - | (name + equals.suppress() + test + arg_comma.suppress())("default") - | (star.suppress() + name + arg_comma.suppress())("star") - | (name + colon.suppress() + typedef_test + equals.suppress() + test + arg_comma.suppress())("type default") - | (name + colon.suppress() + typedef_test + arg_comma.suppress())("type"), + data_args = Group( + Optional( + lparen.suppress() + ZeroOrMore( + Group( + # everything here must end with arg_comma + (name + arg_comma.suppress())("name") + | (name + equals.suppress() + test + arg_comma.suppress())("default") + | (star.suppress() + name + arg_comma.suppress())("star") + | (name + colon.suppress() + typedef_test + equals.suppress() + test + arg_comma.suppress())("type default") + | (name + colon.suppress() + typedef_test + arg_comma.suppress())("type"), + ), + ) + rparen.suppress(), ), - ) + rparen.suppress())) + Optional(keyword("from").suppress() + testlist) - data_suite = Group(colon.suppress() - ( - (newline.suppress() + indent.suppress() + Optional(docstring) + Group(OneOrMore(stmt)) + dedent.suppress())("complex") - | (newline.suppress() + indent.suppress() + docstring + dedent.suppress() | docstring)("docstring") - | simple_stmt("simple") - ) | newline("empty")) + ) + Optional(keyword("from").suppress() + testlist) + data_suite = Group( + colon.suppress() - ( + (newline.suppress() + indent.suppress() + Optional(docstring) + Group(OneOrMore(stmt)) + dedent.suppress())("complex") + | (newline.suppress() + indent.suppress() + docstring + dedent.suppress() | docstring)("docstring") + | simple_stmt("simple") + ) | newline("empty"), + ) datadef_ref = keyword("data").suppress() + name + data_args + data_suite match_datadef = Forward() @@ -1687,11 +1737,13 @@ class Grammar(object): | basic_stmt + end_simple_stmt_item | destructuring_stmt + end_simple_stmt_item, ) - simple_stmt <<= trace(condense( - simple_stmt_item - + ZeroOrMore(fixto(semicolon, "\n") + simple_stmt_item) - + (newline | endline_semicolon), - )) + simple_stmt <<= trace( + condense( + simple_stmt_item + + ZeroOrMore(fixto(semicolon, "\n") + simple_stmt_item) + + (newline | endline_semicolon), + ), + ) stmt <<= final(trace(compound_stmt | simple_stmt)) base_suite <<= condense(newline + indent - OneOrMore(stmt) - dedent) simple_suite = attach(stmt, make_suite_handle) @@ -1729,15 +1781,19 @@ class Grammar(object): rest_of_arg = ZeroOrMore(parens | brackets | braces | ~comma + ~rparen + any_char) tfpdef_tokens = base_name - Optional(originalTextFor(colon - rest_of_arg)) tfpdef_default_tokens = base_name - Optional(originalTextFor((equals | colon) - rest_of_arg)) - parameters_tokens = Group(Optional(tokenlist( - Group( - dubstar - tfpdef_tokens - | star - Optional(tfpdef_tokens) - | slash - | tfpdef_default_tokens, - ) + Optional(passthrough.suppress()), - comma + Optional(passthrough), # implicitly suppressed - ))) + parameters_tokens = Group( + Optional( + tokenlist( + Group( + dubstar - tfpdef_tokens + | star - Optional(tfpdef_tokens) + | slash + | tfpdef_default_tokens, + ) + Optional(passthrough.suppress()), + comma + Optional(passthrough), # implicitly suppressed + ), + ), + ) split_func = attach( start_marker.suppress() diff --git a/coconut/compiler/matching.py b/coconut/compiler/matching.py index ac2af8890..4d8cea8ea 100644 --- a/coconut/compiler/matching.py +++ b/coconut/compiler/matching.py @@ -294,8 +294,10 @@ def match_in_args_kwargs(self, pos_only_match_args, match_args, args, kwargs, al self.add_def( tempvar + " = " + args + "[" + str(i) + "] if _coconut.len(" + args + ") > " + str(i) + " else " - + "".join(kwargs + '.pop("' + name + '") if "' + name + '" in ' + kwargs + " else " - for name in names[:-1]) + + "".join( + kwargs + '.pop("' + name + '") if "' + name + '" in ' + kwargs + " else " + for name in names[:-1] + ) + kwargs + '.pop("' + names[-1] + '")', ) to_match.append((True, match, tempvar)) diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 51e11ccb0..e94983d0d 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -79,10 +79,14 @@ def find_new_value(value, toklist, new_toklist): try: return new_toklist[toklist.index(value)] except ValueError: - complain(lambda: CoconutInternalException("inefficient reevaluation of tokens: {} not in {}".format( - value, - toklist, - ))) + complain( + lambda: CoconutInternalException( + "inefficient reevaluation of tokens: {} not in {}".format( + value, + toklist, + ), + ), + ) return evaluate_tokens(value) diff --git a/setup.py b/setup.py index a8badc36b..6a578e484 100644 --- a/setup.py +++ b/setup.py @@ -62,10 +62,12 @@ author_email=author_email, install_requires=requirements, extras_require=extras, - packages=setuptools.find_packages(exclude=[ - "docs", - "tests", - ]), + packages=setuptools.find_packages( + exclude=[ + "docs", + "tests", + ], + ), include_package_data=True, zip_safe=False, entry_points={ From e3812b8f5a40eaa72ba8d446546586d9b61302b5 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 Jul 2019 15:10:46 -0700 Subject: [PATCH 153/163] Set version to 1.4.1 --- coconut/root.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coconut/root.py b/coconut/root.py index 3360b4971..c5fc9d70f 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -23,10 +23,10 @@ # VERSION: # ----------------------------------------------------------------------------------------------------------------------- -VERSION = "1.4.0" +VERSION = "1.4.1" VERSION_NAME = "Ernest Scribbler" # False for release, int >= 1 for develop -DEVELOP = 43 +DEVELOP = False # ----------------------------------------------------------------------------------------------------------------------- # CONSTANTS: From f3a6703cabf0a58eb0c51fb9abc11575ef866c12 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 Jul 2019 15:42:52 -0700 Subject: [PATCH 154/163] Fix MyPy errors --- Makefile | 2 +- coconut/compiler/compiler.py | 13 ++++++++++--- coconut/stubs/__coconut__.pyi | 2 +- tests/main_test.py | 10 +++++----- tests/src/cocotest/agnostic/util.coco | 8 ++++---- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index f4fc775a7..e6b9b3c7c 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ test-tests: # same as test-basic but also runs mypy .PHONY: test-mypy test-mypy: - python ./tests --strict --force --target sys --mypy --follow-imports silent --ignore-missing-imports + python ./tests --strict --force --target sys --line-numbers --mypy --follow-imports silent --ignore-missing-imports python ./tests/dest/runner.py python ./tests/dest/extras.py diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 03d085e8c..b95daae5f 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -424,9 +424,12 @@ def setup(self, target=None, strict=False, minify=False, line_numbers=False, kee extra="supported targets are " + ', '.join(ascii(t) for t in specific_targets) + ", or leave blank for universal", ) logger.log_vars("Compiler args:", locals()) - self.target, self.strict, self.minify, self.line_numbers, self.keep_lines, self.no_tco = ( - target, strict, minify, line_numbers, keep_lines, no_tco, - ) + self.target = target + self.strict = strict + self.minify = minify + self.line_numbers = line_numbers + self.keep_lines = keep_lines + self.no_tco = no_tco def __reduce__(self): """Return pickling information.""" @@ -479,6 +482,8 @@ def reset(self): @contextmanager def inner_environment(self): """Set up compiler to evaluate inner expressions.""" + line_numbers, self.line_numbers = self.line_numbers, False + keep_lines, self.keep_lines = self.keep_lines, False comments, self.comments = self.comments, {} skips, self.skips = self.skips, [] docstring, self.docstring = self.docstring, "" @@ -486,6 +491,8 @@ def inner_environment(self): try: yield finally: + self.line_numbers = line_numbers + self.keep_lines = keep_lines self.comments = comments self.skips = skips self.docstring = docstring diff --git a/coconut/stubs/__coconut__.pyi b/coconut/stubs/__coconut__.pyi index 06fe1cd57..e98e796bb 100644 --- a/coconut/stubs/__coconut__.pyi +++ b/coconut/stubs/__coconut__.pyi @@ -93,7 +93,7 @@ starmap = _coconut.itertools.starmap if sys.version_info >= (3, 2): - memoize = _coconut.functools.lru_cache + from functools import lru_cache as memoize else: from backports.functools_lru_cache import lru_cache as memoize # type: ignore _coconut.functools.lru_cache = memoize # type: ignore diff --git a/tests/main_test.py b/tests/main_test.py index e711a95a4..b4f82710c 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -461,6 +461,9 @@ def test_no_tco(self): def test_strict(self): run(["--strict"]) + def test_line_numbers(self): + run(["--line-numbers"]) + def test_run(self): run(use_run_arg=True) @@ -468,14 +471,11 @@ def test_run(self): def test_jobs_zero(self): run(["--jobs", "0"]) - def test_simple_line_numbers(self): - run_runnable(["-n", "--linenumbers"]) - def test_simple_keep_lines(self): - run_runnable(["-n", "--keeplines"]) + run_runnable(["-n", "--keep-lines"]) def test_simple_line_numbers_keep_lines(self): - run_runnable(["-n", "--linenumbers", "--keeplines"]) + run_runnable(["-n", "--line-numbers", "--keep-lines"]) def test_simple_minify(self): run_runnable(["-n", "--minify"]) diff --git a/tests/src/cocotest/agnostic/util.coco b/tests/src/cocotest/agnostic/util.coco index 9c2b01a4d..9f18f99d6 100644 --- a/tests/src/cocotest/agnostic/util.coco +++ b/tests/src/cocotest/agnostic/util.coco @@ -25,7 +25,7 @@ plus1sq_1 = square..plus1 # type: ignore sqplus1_1 = plus1 sqplus1_1 ..= square # type: ignore -plus1sq_2 = (x) -> x |> plus1 |> square +plus1sq_2 = (x) -> x |> plus1 |> square # type: ignore sqplus1_2 = (x) -> x |> square |> plus1 plus1sq_3 = square <.. plus1 @@ -138,7 +138,7 @@ def qsort4(l: int[]) -> int[]: return None # type: ignore def qsort5(l: int$[]) -> int$[]: """Iterator Match Quick Sort.""" - match (head,) :: tail in l: + match (head,) :: tail in l: # type: ignore tail, tail_ = tee(tail) return (qsort5((x for x in tail if x <= head)) :: (head,) # The pivot is a tuple @@ -147,8 +147,8 @@ def qsort5(l: int$[]) -> int$[]: else: return iter(()) def qsort6(l: int$[]) -> int$[]: - match [head] :: tail in l: - tail = reiterable(tail) + match [head] :: tail in l: # type: ignore + tail = reiterable(tail) # type: ignore yield from ( qsort6(x for x in tail if x <= head) :: (head,) From 155f18e46dbcf4d97bb12c510d325c532362b023 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 Jul 2019 15:45:36 -0700 Subject: [PATCH 155/163] Fix AppVeyor --- .appveyor.yml | 2 +- Makefile | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index fc9369701..69b6f7d5c 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -28,7 +28,7 @@ environment: install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;c:\\MinGW\\bin;%PATH%" - "copy c:\\MinGW\\bin\\mingw32-make.exe c:\\MinGW\\bin\\make.exe" - - make install + - make install-user build: false diff --git a/Makefile b/Makefile index e6b9b3c7c..5579ac862 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,11 @@ install: pip install --upgrade setuptools pip pip install .[tests] +.PHONY: install-user +install-user: + pip install --user --upgrade setuptools pip + pip install --user .[tests] + .PHONY: dev dev: pip install --upgrade setuptools pip From 52cadbf47ad195f58676a7ad3a30974ffd81fe79 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 Jul 2019 15:47:26 -0700 Subject: [PATCH 156/163] Remove 2.6 test --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0aa6b1217..070978406 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ notifications: sudo: false cache: pip python: -- '2.6' - '2.7' - pypy - '3.5' From f9d665587cfdf0137757278898739f35c8b2f277 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 Jul 2019 15:48:29 -0700 Subject: [PATCH 157/163] Further fix AppVeyor --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5579ac862..8b469458d 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ format: dev # test-all takes a very long time and should usually only be run by Travis .PHONY: test-all test-all: - pytest --strict -s tests + pytest --strict -s ./tests # for quickly testing nearly everything locally, just use test-basic .PHONY: test-basic From c17bf08a2fa620e072994a9a255da9c70362ede9 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 Jul 2019 16:16:29 -0700 Subject: [PATCH 158/163] Fix AppVeyor PATH --- .appveyor.yml | 2 +- CONTRIBUTING.md | 2 +- coconut/compiler/compiler.py | 11 ++++------- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 69b6f7d5c..cbf56c1bf 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -26,7 +26,7 @@ environment: PYTHON_ARCH: "32" install: - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;c:\\MinGW\\bin;%PATH%" + - "SET PATH=%APPDATA%\\Python;%APPDATA%\\Python\\Scripts;%PYTHON%;%PYTHON%\\Scripts;c:\\MinGW\\bin;%PATH%" - "copy c:\\MinGW\\bin\\mingw32-make.exe c:\\MinGW\\bin\\make.exe" - make install-user diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e7c878f3c..8b8a79895 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -162,7 +162,7 @@ After you've tested your changes locally, you'll want to add more permanent test 1. Run `make check-reqs` and update dependencies as necessary 1. Run `make format` 1. Check changes in [`compiled-cocotest`](https://github.com/evhub/compiled-cocotest), [`pyprover`](https://github.com/evhub/pyprover), and [`coconut-prelude`](https://github.com/evhub/coconut-prelude) - 1. Check [Codacy issues](https://www.codacy.com/app/evanjhub) (for `coconut` and `compiled-cocotest`) and [LGTM alerts](https://lgtm.com/projects/g/evhub/coconut/) + 1. Check [Codebeat](https://codebeat.co/a/evhub/projects) and [LGTM](https://lgtm.com/dashboard) for `coconut` and `compiled-cocotest` 1. Make sure [`coconut-develop`](https://pypi.python.org/pypi/coconut-develop) package looks good 1. Run `make docs` and ensure local documentation looks good 1. Make sure [develop documentation](http://coconut.readthedocs.io/en/develop/) looks good diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index b95daae5f..a13367873 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -833,8 +833,7 @@ def str_proc(self, inputstring, **kwargs): if c == "\n": if len(hold[_start]) == 1: raise self.make_err(CoconutSyntaxError, "linebreak in non-multiline string", inputstring, x, reformat=False) - else: - skips = addskip(skips, self.adjust(lineno(x, inputstring))) + skips = addskip(skips, self.adjust(lineno(x, inputstring))) hold[_contents] += hold[_stop] + c hold[_stop] = None elif count_end(hold[_contents], "\\") % 2 == 1: @@ -850,8 +849,7 @@ def str_proc(self, inputstring, **kwargs): if c == "\n": if len(hold[_start]) == 1: raise self.make_err(CoconutSyntaxError, "linebreak in non-multiline string", inputstring, x, reformat=False) - else: - skips = addskip(skips, self.adjust(lineno(x, inputstring))) + skips = addskip(skips, self.adjust(lineno(x, inputstring))) hold[_contents] += c elif found is not None: if c == found[0]: @@ -859,9 +857,8 @@ def str_proc(self, inputstring, **kwargs): elif len(found) == 1: # found == "_" if c == "\n": raise self.make_err(CoconutSyntaxError, "linebreak in non-multiline string", inputstring, x, reformat=False) - else: - hold = [c, found, None] # [_contents, _start, _stop] - found = None + hold = [c, found, None] # [_contents, _start, _stop] + found = None elif len(found) == 2: # found == "__" out.append(self.wrap_str("", found[0], False)) found = None From e8250f3af7407c5af86cbfb334e7324507f342aa Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 Jul 2019 16:36:31 -0700 Subject: [PATCH 159/163] Remove targetless MyPy test --- tests/main_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/main_test.py b/tests/main_test.py index b4f82710c..7103c5cc2 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -440,9 +440,6 @@ def test_normal(self): def test_mypy_snip(self): call(["coconut", "-c", mypy_snip, "--mypy"], assert_output=mypy_snip_err, check_mypy=False, expect_retcode=1) - def test_mypy(self): - run(["--mypy"] + mypy_args, expect_retcode=None) # fails due to tutorial mypy errors - def test_mypy_sys(self): run(["--mypy"] + mypy_args, agnostic_target="sys", expect_retcode=None) # fails due to tutorial mypy errors From a6fddc092229e1eafaa50eceb187b330a6b89724 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 28 Jul 2019 13:08:16 -0700 Subject: [PATCH 160/163] Fix AppVeyor installation --- .appveyor.yml | 3 ++- Makefile | 5 ----- coconut/constants.py | 3 ++- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index cbf56c1bf..9e43f21f9 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -28,7 +28,8 @@ environment: install: - "SET PATH=%APPDATA%\\Python;%APPDATA%\\Python\\Scripts;%PYTHON%;%PYTHON%\\Scripts;c:\\MinGW\\bin;%PATH%" - "copy c:\\MinGW\\bin\\mingw32-make.exe c:\\MinGW\\bin\\make.exe" - - make install-user + - python -m pip install --user --upgrade setuptools pip + - python -m pip install .[tests] build: false diff --git a/Makefile b/Makefile index 8b469458d..57fc22cc1 100644 --- a/Makefile +++ b/Makefile @@ -3,11 +3,6 @@ install: pip install --upgrade setuptools pip pip install .[tests] -.PHONY: install-user -install-user: - pip install --user --upgrade setuptools pip - pip install --user .[tests] - .PHONY: dev dev: pip install --upgrade setuptools pip diff --git a/coconut/constants.py b/coconut/constants.py index e3704b974..20eff4ff7 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -176,7 +176,6 @@ def checksum(data): "pyparsing": (2, 4, 0), "cPyparsing": (2, 4, 0, 1, 0, 0), "pre-commit": (1,), - "pygments": (2, 4), "recommonmark": (0, 5), "psutil": (5,), "jupyter": (1, 0), @@ -192,6 +191,8 @@ def checksum(data): ("ipython", "py3"): (7, 6), ("jupyter-console", "py3"): (6,), ("ipykernel", "py3"): (5, 1), + # don't upgrade this; it breaks with Python 3.4 on Windows + "pygments": (2, 3, 1), # don't upgrade this to allow all versions "prompt_toolkit:3": (1,), # don't upgrade this; it breaks on Python 2.6 From 082c7d424f9f297cd6aedaeea98edfeb884a5f9f Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 28 Jul 2019 14:15:30 -0700 Subject: [PATCH 161/163] Fix py34 reqs --- coconut/constants.py | 10 ++++++---- coconut/requirements.py | 5 +++++ tests/src/extras.coco | 4 +++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index 20eff4ff7..e81466534 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -168,7 +168,8 @@ def checksum(data): "tests": ( "pytest", "pexpect", - "numpy", + ("numpy", "py34"), + ("numpy", "py2"), ), } @@ -187,11 +188,12 @@ def checksum(data): "watchdog": (0, 9), "trollius": (2, 2), "requests": (2,), - "numpy": (1,), - ("ipython", "py3"): (7, 6), + ("numpy", "py34"): (1,), + ("numpy", "py2"): (1,), ("jupyter-console", "py3"): (6,), ("ipykernel", "py3"): (5, 1), - # don't upgrade this; it breaks with Python 3.4 on Windows + # don't upgrade these; they break with Python 3.4 on Windows + ("ipython", "py3"): (6, 5), "pygments": (2, 3, 1), # don't upgrade this to allow all versions "prompt_toolkit:3": (1,), diff --git a/coconut/requirements.py b/coconut/requirements.py index eb009ea27..92aeb1c24 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -77,6 +77,11 @@ def get_reqs(which): req_str += ";python_version>='3'" elif PY2: continue + elif env_marker == "py34": + if supports_env_markers: + req_str += ";python_version>='3.4'" + elif not PY34: + continue else: raise ValueError("unknown env marker id " + repr(env_marker)) reqs.append(req_str) diff --git a/tests/src/extras.coco b/tests/src/extras.coco index 5c4531069..210c6ab8f 100644 --- a/tests/src/extras.coco +++ b/tests/src/extras.coco @@ -2,6 +2,8 @@ from coconut.__coconut__ import consume as coc_consume # type: ignore from coconut.constants import ( IPY, + PY2, + PY34, PY35, WINDOWS, PYPY, @@ -149,7 +151,7 @@ def test_extras(): assert "map" in keyword_complete_result["matches"] assert keyword_complete_result["cursor_start"] == 0 assert keyword_complete_result["cursor_end"] == 1 - if not PYPY: + if not PYPY and (PY2 or PY34): import numpy as np assert np.all(fmap(-> _ + 1, np.arange(3)) == np.array([1, 2, 3])) print("") From 306463835f339ea1acee93b6b50e7f1157ebaccd Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 29 Jul 2019 16:34:53 -0700 Subject: [PATCH 162/163] Further fix AppVeyor req issues --- .appveyor.yml | 3 --- coconut/constants.py | 7 +++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 9e43f21f9..aab96fb31 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -12,9 +12,6 @@ environment: - PYTHON: "C:\\Python27" PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "32" - - PYTHON: "C:\\Python33" - PYTHON_VERSION: "3.3.x" - PYTHON_ARCH: "32" - PYTHON: "C:\\Python34" PYTHON_VERSION: "3.4.x" PYTHON_ARCH: "32" diff --git a/coconut/constants.py b/coconut/constants.py index e81466534..aecfda30a 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -138,12 +138,11 @@ def checksum(data): ), "jupyter": ( "jupyter", + "jupyter-console", ("ipython", "py2"), ("ipython", "py3"), ("ipykernel", "py2"), ("ipykernel", "py3"), - ("jupyter-console", "py2"), - ("jupyter-console", "py3"), ), "mypy": ( "mypy", @@ -190,8 +189,9 @@ def checksum(data): "requests": (2,), ("numpy", "py34"): (1,), ("numpy", "py2"): (1,), - ("jupyter-console", "py3"): (6,), ("ipykernel", "py3"): (5, 1), + # don't upgrade this; it breaks on Python 2 and Python 3.4 on Windows + "jupyter-console": (5, 2), # don't upgrade these; they break with Python 3.4 on Windows ("ipython", "py3"): (6, 5), "pygments": (2, 3, 1), @@ -203,7 +203,6 @@ def checksum(data): "vprof": (0, 36), # don't upgrade these; they break on Python 2 ("ipython", "py2"): (5, 4), - ("jupyter-console", "py2"): (5, 2), ("ipykernel", "py2"): (4, 10), "prompt_toolkit:2": (1,), # don't upgrade these; they break on master From 309e56b001ac663a8893a185650e1412e98e04ea Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 6 Aug 2019 16:49:08 -0700 Subject: [PATCH 163/163] Fix windows tests --- tests/main_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/main_test.py b/tests/main_test.py index 7103c5cc2..4c4621d9d 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -326,7 +326,7 @@ def run_pyprover(**kwargs): def comp_prelude(args=[], **kwargs): """Compiles evhub/coconut-prelude.""" call(["git", "clone", prelude_git]) - if PY36: + if PY36 and not WINDOWS: args.extend(["--target", "3.6", "--mypy"]) call_coconut([os.path.join(prelude, "setup.coco"), "--strict"] + args, **kwargs) call_coconut([os.path.join(prelude, "prelude-source"), os.path.join(prelude, "prelude"), "--strict"] + args, **kwargs) @@ -399,7 +399,7 @@ def test_compile_to_file(self): call_coconut([runnable_coco, runnable_py]) call_python([runnable_py, "--arg"], assert_output=True) - if IPY: + if IPY and (not WINDOWS or PY35): def test_ipython_extension(self): call( ["ipython", "--ext", "coconut", "-c", r'%coconut ' + coconut_snip],