From 72e2b9b4c3baec37b14899b91a702c717222d132 Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Fri, 14 Mar 2025 09:05:08 -0500 Subject: [PATCH 1/9] style: lint markdown files --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b8f2bf32..5d6f306f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,6 +22,11 @@ repos: hooks: - id: codespell args: [--write-changes] + # lint markdown + - repo: https://github.com/DavidAnson/markdownlint-cli2 + rev: v0.17.2 + hooks: + - id: markdownlint-cli2 - repo: local hooks: - id: cep-header From 4407de22d27831903405551d04cca54814464f1b Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Fri, 14 Mar 2025 09:15:19 -0500 Subject: [PATCH 2/9] fix: add config file --- .markdownlint.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .markdownlint.yaml diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 00000000..85d3791f --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,8 @@ +# MD033/no-inline-html : Inline HTML : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md033.md +MD033: + # Allowed elements + allowed_elements: + - table + - tr + - td + - br From 5da5d0e732bbbeb56121c6550e0ff61960cf0a0b Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Fri, 14 Mar 2025 09:17:52 -0500 Subject: [PATCH 3/9] Add line length rule to markdownlint configuration --- .markdownlint.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.markdownlint.yaml b/.markdownlint.yaml index 85d3791f..e45e95e3 100644 --- a/.markdownlint.yaml +++ b/.markdownlint.yaml @@ -6,3 +6,8 @@ MD033: - tr - td - br + +# MD013/line-length : Line length : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md013.md +MD013: + # Number of characters + line_length: 120 From b616ca2fa039d637c206f73689e581aa6674915f Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Fri, 14 Mar 2025 09:19:26 -0500 Subject: [PATCH 4/9] Add --fix argument to markdownlint-cli2 --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d6f306f..eb121eb2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,7 @@ repos: rev: v0.17.2 hooks: - id: markdownlint-cli2 + args: [ --fix ] - repo: local hooks: - id: cep-header From d32a1d9c80499be43642f426b45da43bfc31463a Mon Sep 17 00:00:00 2001 From: beckermr Date: Fri, 14 Mar 2025 09:52:58 -0500 Subject: [PATCH 5/9] fix: more config --- .markdownlint.yaml | 2 ++ .pre-commit-config.yaml | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.markdownlint.yaml b/.markdownlint.yaml index e45e95e3..23b3d82e 100644 --- a/.markdownlint.yaml +++ b/.markdownlint.yaml @@ -6,6 +6,8 @@ MD033: - tr - td - br + - ul + - li # MD013/line-length : Line length : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md013.md MD013: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb121eb2..70043fb9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,8 @@ repos: rev: v0.17.2 hooks: - id: markdownlint-cli2 - args: [ --fix ] + args: ['--fix'] + exclude: ^.github/.*\.md$ - repo: local hooks: - id: cep-header From fdc7a2631777a6cfabafebcfc1e677677139d1fa Mon Sep 17 00:00:00 2001 From: beckermr Date: Fri, 14 Mar 2025 09:56:56 -0500 Subject: [PATCH 6/9] fix: more chars --- .markdownlint.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.markdownlint.yaml b/.markdownlint.yaml index 23b3d82e..df03f7a9 100644 --- a/.markdownlint.yaml +++ b/.markdownlint.yaml @@ -12,4 +12,6 @@ MD033: # MD013/line-length : Line length : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md013.md MD013: # Number of characters - line_length: 120 + # we have set this to be very high right now due to the linter not + # ignoring the limit for inline html and tables + line_length: 500 From 8495acda1078ce07d7bee536940548248a7abb07 Mon Sep 17 00:00:00 2001 From: beckermr Date: Fri, 14 Mar 2025 10:02:10 -0500 Subject: [PATCH 7/9] fix: more config for inline html we want --- .markdownlint.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.markdownlint.yaml b/.markdownlint.yaml index df03f7a9..69f97754 100644 --- a/.markdownlint.yaml +++ b/.markdownlint.yaml @@ -8,6 +8,11 @@ MD033: - br - ul - li + - code + - a + - details + - summary + - img # MD013/line-length : Line length : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md013.md MD013: From 134e72a17c39c12c8b53a0eb9f62132784400516 Mon Sep 17 00:00:00 2001 From: beckermr Date: Fri, 14 Mar 2025 15:16:56 -0500 Subject: [PATCH 8/9] style: fix style in older CEPs --- README.md | 52 +++++++++++++++++++++++------------------------ cep-0001.md | 16 +++++++-------- cep-0003.md | 1 - cep-0004.md | 13 ++++++------ cep-0006.md | 15 +++++--------- cep-0008.md | 1 + cep-0010.md | 11 ++++------ cep-0011.md | 58 ++++++++++++++++++++++++++--------------------------- cep-0012.md | 26 +++++++++++------------- cep-0013.md | 6 ++++-- cep-0014.md | 27 +++++++++++-------------- cep-0015.md | 4 ++-- cep-0016.md | 53 ++++++++++++++++++++++++------------------------ cep-0017.md | 2 +- cep-0019.md | 16 ++++++++------- cep-0020.md | 9 +++++---- 16 files changed, 150 insertions(+), 160 deletions(-) diff --git a/README.md b/README.md index d980223b..7d9a1743 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -## Conda Enhancement Proposals +# Conda Enhancement Proposals [Conda](https://docs.conda.io/) is an Open Source project with a diverse and wide-ranging ecosystem. @@ -6,31 +6,31 @@ To better allow community members to provide feedback and proposals for conda's implementation, all major changes should be submitted as **Conda Enhancement Proposals (CEP)**. -## Accepted CEPS: - -| CEP | Title | -| --- | ------- | -| [0000](cep-0000.md) | CEP template | -| [0001](cep-0001.md) | CEP Purpose and Guidelines | -| [0002](cep-0002.md) | Add plugin architecture to conda | -| [0003](cep-0003.md) | Using the Mamba solver in conda | -| [0004](cep-0004.md) | Implement initial conda plugin mechanism | -| 0005 | _Does not exist_ | -| [0006](cep-0006.md) | Add Channel Notices to conda -| 0007 | _Does not exist_ | -| [0008](cep-0008.md) | Conda Release Schedule | -| [0009](cep-0009.md) | Conda Deprecation Schedule | -| [0010](cep-0010.md) | Conda Version Support | -| [0011](cep-0011.md) | Define the menuinst standard | -| [0012](cep-0012.md) | Serving run_exports metadata in conda channels | -| [0013](cep-0013.md) | A new recipe format – part 1 | -| [0014](cep-0014.md) | A new recipe format – part 2 - the allowed keys & values | +## Accepted CEPS + +| CEP | Title | +|---------------------|-----------------------------------------------------------------------------| +| [0000](cep-0000.md) | CEP template | +| [0001](cep-0001.md) | CEP Purpose and Guidelines | +| [0002](cep-0002.md) | Add plugin architecture to conda | +| [0003](cep-0003.md) | Using the Mamba solver in conda | +| [0004](cep-0004.md) | Implement initial conda plugin mechanism | +| 0005 | _Does not exist_ | +| [0006](cep-0006.md) | Add Channel Notices to conda | +| 0007 | _Does not exist_ | +| [0008](cep-0008.md) | Conda Release Schedule | +| [0009](cep-0009.md) | Conda Deprecation Schedule | +| [0010](cep-0010.md) | Conda Version Support | +| [0011](cep-0011.md) | Define the menuinst standard | +| [0012](cep-0012.md) | Serving run_exports metadata in conda channels | +| [0013](cep-0013.md) | A new recipe format – part 1 | +| [0014](cep-0014.md) | A new recipe format – part 2 - the allowed keys & values | | [0015](cep-0015.md) | Hosting repodata.json and packages separately by adding a base_url property | -| [0016](cep-0016.md) | Sharded Repodata | -| [0017](cep-0017.md) | Optional python site-packages path in repodata | -| [0018](cep-0018.md) | Migration to the Zulip chat platform | -| [0019](cep-0019.md) | Computing the hash of the contents in a directory | -| [0020](cep-0020.md) | Support for `abi3` Python packages | +| [0016](cep-0016.md) | Sharded Repodata | +| [0017](cep-0017.md) | Optional python site-packages path in repodata | +| [0018](cep-0018.md) | Migration to the Zulip chat platform | +| [0019](cep-0019.md) | Computing the hash of the contents in a directory | +| [0020](cep-0020.md) | Support for `abi3` Python packages | ## References @@ -39,7 +39,7 @@ These proposals are similar to conda-forge's [CFEP](https://github.com/conda-for ## Writing a new CEP -Community members are encouraged to author a CEP to suggest changes *before* +Community members are encouraged to author a CEP to suggest changes _before_ any code is written to allow for the community to discuss the proposed changes. The formal process by which CEPs should be authored and how they are reviewed diff --git a/cep-0001.md b/cep-0001.md index f3137f32..7ca9c108 100644 --- a/cep-0001.md +++ b/cep-0001.md @@ -10,14 +10,14 @@ Implementation NA -# Abstract +## Abstract CEPs are the manner in which the conda project community suggests changes to the software project, and outlines how the suggested changes are reviewed. This document provides details on what a CEP is, how to author one, and the workflow used to review these proposals. -# What is a CEP +## What is a CEP CEP stands for "Conda Enhancement Proposal". A CEP is a document which outlines a suggested change to how the project works. These changes may be @@ -33,7 +33,7 @@ CEPs are maintained as text files using Markdown for formatting in the [conda/ceps](https://github.com/conda/ceps) repository. Their revision history is a historical record of the proposed change. -# Workflow +## Workflow The CEP process begins with an idea for a change in how the conda project works. These changes can be technical or address the social aspects of the @@ -61,14 +61,14 @@ template in [CEP 0](cep-0000.md) and described below. While working on the CEP prior to submission, please set its status to *draft*. -## Submission +### Submission Once complete, this draft should be submitted as a pull request to the [conda/ceps](https://github.com/conda/ceps) repository with the status changed to *proposed*. At this point, members of the conda community will review the submission. -## Review +### Review CEP review will begin once a pull request has been made. All members of the conda community are welcome and encouraged to participate in the review @@ -76,7 +76,7 @@ of CEPs in a civil and respectful manner. The CEP champion(s) should be prepared to answer questions on the proposal and make updates to the proposal as recommended by the community. -## Resolution +### Resolution All CEPs will be resolved as either *rejected*, *accepted* or *deferred* depending on the consensus of the community. Once the community has reached a @@ -93,7 +93,7 @@ changes have been implemented. *Deferred* CEPs can be re-submitted in a new pull request if the circumstances that justified deferring instead of accepting or rejecting have changed. -# CEP Content +## CEP Content All CEPs should begin with a top level table with the following information: @@ -129,7 +129,7 @@ A final **copyright** section is also required. CEP is to be pronounced `/sep/`. -If you still have doubts about how to actually pronounce it, here is a reference (we accept UK and US pronunciations): https://dictionary.cambridge.org/dictionary/english/cep +If you still have doubts about how to actually pronounce it, here is a reference (we accept UK and US pronunciations): ## Conda capitalization standards diff --git a/cep-0003.md b/cep-0003.md index aa4a5c67..56a28ab4 100644 --- a/cep-0003.md +++ b/cep-0003.md @@ -138,7 +138,6 @@ CEP, yet to be drafted. - [conda-libmamba-solver]: The experimental libmamba solver integrations, packaged separately - [CEP 2]: https://github.com/conda/ceps/pull/1 [libsolv]: https://github.com/openSUSE/libsolv [pycosat]: https://pypi.org/project/pycosat/ diff --git a/cep-0004.md b/cep-0004.md index 4a86ccf7..69ad0f96 100644 --- a/cep-0004.md +++ b/cep-0004.md @@ -26,12 +26,12 @@ Plugins in `conda` will integrate the "hook + entry point" structure by utilizin - Define the hook(s) to be registered - Register the plugin under the `conda` entrypoint namespace - ### Hook Below is an example of a very basic plugin "hook": -_my_plugin.py_ +#### my_plugin.py + ```python import conda.plugins @@ -43,10 +43,10 @@ def conda_subcommands(): ### Packaging via a `pyproject.toml` file - Below is an example that configures `setuptools` using a `pyproject.toml` file (note that the `setup.py` file is optional if a `pyproject.toml` file is defined): -_pyproject.toml_ +#### pyproject.toml + ```toml [build-system] requires = ["setuptools", "setuptools-scm"] @@ -63,12 +63,12 @@ dependencies = ["conda"] my-conda-plugin = "my_plugin" ``` - ### Packaging via a `setup.py` file Below is an example of an entry point namespace for the custom plugin function, decorated with the plugin hook shown in the "Hook" section above: -_setup.py_ +#### setup.py + ```python from setuptools import setup @@ -80,7 +80,6 @@ setup( ) ``` - ## Rationale This new `conda` plugin API ecosystem will bring about many benefits and possibilities, including but not limited to: diff --git a/cep-0006.md b/cep-0006.md index d1117c87..f524acb3 100644 --- a/cep-0006.md +++ b/cep-0006.md @@ -10,7 +10,6 @@ Implementation https://github.com/conda/conda/pull/11462 - ## Abstract In order to facilitate better communication between users of `conda` and channel owners, we want to @@ -49,7 +48,7 @@ traffic to occur, so the requirement for having an active internet connection is Here's an example of how a notification message may appear while running the `conda create` command: -``` +```bash $ conda create -n geopandas geopandas Collecting package metadata (repodata.json): done Solving environment: done @@ -78,7 +77,7 @@ The following are a couple examples to show exactly how it would function: **Basic usage:** grabs notices for all current channels: -``` +```bash $ conda notices Retrieving notices: done @@ -96,7 +95,7 @@ Channel "conda-forge" has the following notices: **Show a single channel:** grabs notices for a single channel: -``` +```bash $ conda notices -c defaults Channel "defaults" has the following notices: @@ -161,7 +160,6 @@ Additionally, this new notification space can also provide a place for us to rel to a more visible spot (at the end of command output versus in the middle of the output). On top of this, other channels can use these notices as a way to share news with their users or requests for help in maintaining their channels. - ## Rationale In order to keep this feature as a simple as possible and reduce the number of moving parts, we choose to serve @@ -175,12 +173,10 @@ these messages if they do not want to see them while running commands such as "i we feel these messages are important for users to see, we also do not want to clutter their terminal output. Ultimately, this is why we are choosing to turn this on by default but are also providing a way to disable it. - ## Backwards Compatibility We do not expect any backwards compatibility issues for this new feature. - ## Alternatives - **Show notices at the beginning of environment activation:** This was deemed too intrusive/annoying. @@ -200,12 +196,11 @@ We do not expect any backwards compatibility issues for this new feature. it makes more sense to put these in their own file. Plus, it allows more flexibility for dynamic routing options to this file if that becomes necessary in the future. - ## References This implementation is similar to the way that `npm` handles version update reminders: -``` +```bash npm install travishathaway@thath-work-laptop up to date, audited 365 packages in 3s @@ -230,7 +225,7 @@ npm notice Pull request to the full implementation: -https://github.com/conda/conda/pull/11462 + Please add any implementation related suggestions for improvements to this pull request. diff --git a/cep-0008.md b/cep-0008.md index 48c2fe9d..e3b283b6 100644 --- a/cep-0008.md +++ b/cep-0008.md @@ -22,6 +22,7 @@ We propose regularly scheduled bi-monthly (every two months) releases where the In a nod to our many different kinds of users, we will also propose a deprecation policy (to be defined in a later CEP) that allows for a slower adoption rate (i.e., users could update every 3-4 months instead). To accomplish better, more predictable versioning, we will adopt [CalVer](https://calver.org/): + - `YY`: the major version will be the shortened year (22+) - `MM`: the minor version will be the shortened month (1-12) - `MICRO`: the micro/patch version is reset to zero every month and incremented for every additional release that occurs during that month (0+) diff --git a/cep-0010.md b/cep-0010.md index 9b37f3e6..e786b2a5 100644 --- a/cep-0010.md +++ b/cep-0010.md @@ -35,7 +35,6 @@ The conda project already commits to maintain backwards compatibility per CEP 9. Any breaking changes will be announced ahead of time and go through our established [deprecation schedule][deprecation-schedule]. - ## Motivation The primary motivation for this CEP is setting clear expectations about how long @@ -49,7 +48,7 @@ of conda. ## Rationale -For many projects (more information here: https://endoflife.date), either +For many projects (more information here: ), either the latest version is supported or the projects have a sliding window of time for their supported versions. This window is a guarantee saying that the version in question will be supported for a specific amount of time. For most who @@ -83,14 +82,12 @@ It has also passed since it recorded 10 "yes" votes and 0 "no" votes giving 10/1 Helpful websites and articles: -- https://endoflife.date -- https://pip.pypa.io/en/latest/development/release-process/#supported-versions -- https://devguide.python.org/versions/ - +- +- +- ## Copyright All CEPs are explicitly [CC0 1.0 Universal](https://creativecommons.org/publicdomain/zero/1.0/). - [deprecation-schedule]: https://github.com/conda-incubator/ceps/blob/main/cep-0009.md diff --git a/cep-0011.md b/cep-0011.md index 83a1f1c5..cbce9f6b 100644 --- a/cep-0011.md +++ b/cep-0011.md @@ -142,9 +142,9 @@ Placeholder | Value `SP_DIR` | Path to Python's `site-packages` directory in `PREFIX`. `HOME` | Path to the user directory (`~`). `ICON_EXT` | Extension of the icon file expected by each platform. `png` in Linux, `icns` in macOS, and `ico` in Windows. Note the dot is _not_ included. -**macOS-only** | +**macOS-only** |
`PYTHONAPP` | Path to the `python` executable installed in `PREFIX`, assuming the `python.app` conda package is installed. Equivalent to `{{ PREFIX }}/python.app/Contents/MacOS/python`. -**Windows-only** | +**Windows-only** |
`SCRIPTS_DIR` | Path to the `Scripts` directory in `PREFIX`. `BASE_PYTHONW` | Path to the `pythonw.exe` executable in `BASE_PREFIX`. `PYTHONW` | Path to the `pythonw.exe` executable in `PREFIX`. @@ -169,7 +169,7 @@ Each platform MUST place the menu artifacts in these target locations: Operating system | Artifact type | User location | System location | Notes -----------------|---------------|---------------|-----------------|------- Linux | `.desktop` file | `~/.local/share/applications` | `/usr/local/share/applications` | Some other user files are modified -macOS | `.app` directory | `~/Applications` | `/Applications` | +macOS | `.app` directory | `~/Applications` | `/Applications` |
Windows | `.lnk` file | `{{ menu_name }}` directory inside Start Menu, Desktop, and/or Quick Launch | Start Menu | These locations are customizable and configured in the Windows registry. - On Linux, little needs to be done because XDG delegates the responsibility to the desktop @@ -233,15 +233,15 @@ See Addendum A below for a best effort in documenting it. ## References -* [Rework linux/osx support plus new simplified format?](https://github.com/conda/menuinst/issues/48) -* [Mamba's implementation in C++](https://github.com/mamba-org/mamba/blob/81a490a/src/core/menuinst.cpp) -* [Interactions between conda, conda-standalone, constructor and menuinst](https://gist.github.com/jaimergp/7de5843421d63fa4a408ac5c8712c3c9) -* [Change the API to `menuinst.install(path_or_dict)`](https://github.com/conda/menuinst/issues/25) -* [`menuinst` wiki as of 2021.10.18](https://github.com/conda/menuinst/wiki/Menu-Shortcut-Config-Structure/632fbc84251c8a8093e1b56b0b5314d23c1e946b) -* [freedesktop.org specification](https://specifications.freedesktop.org/desktop-entry-spec/desktop-entry-spec-latest.html#recognized-keys) -* [Core Foundation Keys (info.plist)](https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html#//apple_ref/doc/uid/TP40009249-SW1) -* [File type association in Windows](https://learn.microsoft.com/en-us/windows/win32/shell/fa-file-types) -* [Default programs in Windows](https://learn.microsoft.com/en-us/windows/win32/shell/default-programs) +- [Rework linux/osx support plus new simplified format?](https://github.com/conda/menuinst/issues/48) +- [Mamba's implementation in C++](https://github.com/mamba-org/mamba/blob/81a490a/src/core/menuinst.cpp) +- [Interactions between conda, conda-standalone, constructor and menuinst](https://gist.github.com/jaimergp/7de5843421d63fa4a408ac5c8712c3c9) +- [Change the API to `menuinst.install(path_or_dict)`](https://github.com/conda/menuinst/issues/25) +- [`menuinst` wiki as of 2021.10.18](https://github.com/conda/menuinst/wiki/Menu-Shortcut-Config-Structure/632fbc84251c8a8093e1b56b0b5314d23c1e946b) +- [freedesktop.org specification](https://specifications.freedesktop.org/desktop-entry-spec/desktop-entry-spec-latest.html#recognized-keys) +- [Core Foundation Keys (info.plist)](https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html#//apple_ref/doc/uid/TP40009249-SW1) +- [File type association in Windows](https://learn.microsoft.com/en-us/windows/win32/shell/fa-file-types) +- [Default programs in Windows](https://learn.microsoft.com/en-us/windows/win32/shell/default-programs) ## Copyright @@ -286,18 +286,18 @@ platform. Currently allowed placeholders are: -* `${PREFIX}`: Python environment prefix -* `${ROOT_PREFIX}`: Python environment prefix of root (conda or otherwise) installation -* `${PYTHON_SCRIPTS}`: Scripts folder in Python environment, `${PREFIX}/Scripts` -* `${MENU_DIR}`: Folder for menu config and icon files, `${PREFIX}/Menu` -* `${PERSONALDIR}`: Not sure -* `${USERPROFILE}`: User's home folder -* `${ENV_NAME}`: The environment in which this shortcut lives. -* `${DISTRIBUTION_NAME}`: The name of the folder of the root prefix, for example "Miniconda" if +- `${PREFIX}`: Python environment prefix +- `${ROOT_PREFIX}`: Python environment prefix of root (conda or otherwise) installation +- `${PYTHON_SCRIPTS}`: Scripts folder in Python environment, `${PREFIX}/Scripts` +- `${MENU_DIR}`: Folder for menu config and icon files, `${PREFIX}/Menu` +- `${PERSONALDIR}`: Not sure +- `${USERPROFILE}`: User's home folder +- `${ENV_NAME}`: The environment in which this shortcut lives. +- `${DISTRIBUTION_NAME}`: The name of the folder of the root prefix, for example "Miniconda" if distribution installed at "C:\Users\Miniconda". -* `${PY_VER}`: The Python major version only. This is taken from the root prefix. Used generally +- `${PY_VER}`: The Python major version only. This is taken from the root prefix. Used generally for placing shortcuts under a menu of a parent installation. -* `${PLATFORM}`: one of (32-bit) or (64-bit). This is taken from the root prefix. Used generally +- `${PLATFORM}`: one of (32-bit) or (64-bit). This is taken from the root prefix. Used generally for placing shortcuts under a menu of a parent installation. #### `MenuItem` on MacOS @@ -312,8 +312,8 @@ Currently allowed placeholders are: Currently allowed placeholders are: -* `${BIN_DIR}`: `PREFIX/bin` -* `${MENU_DIR}`: `PREFIX/Menu` +- `${BIN_DIR}`: `PREFIX/bin` +- `${MENU_DIR}`: `PREFIX/Menu` #### `MenuItem` on Linux @@ -337,14 +337,14 @@ which are replaced by the default Desktop file explorer, and the default web bro Windows has several ways to specify which command should be run with the shortcut: -* `system` + `scriptargument[s]`: path to executable plus its arguments. -* `script` + `scriptargument[s]`: same as above, but the executable is run in a subprocess after +- `system` + `scriptargument[s]`: path to executable plus its arguments. +- `script` + `scriptargument[s]`: same as above, but the executable is run in a subprocess after invoking `ROOT_PYTHON cwp.py PREFIX`. -* `pyscript`: hardcodes `script` to be `PREFIX/python.exe` and takes the value as the first (and +- `pyscript`: hardcodes `script` to be `PREFIX/python.exe` and takes the value as the first (and only) argument. -* `pywscript`: same as above, but uses `pythonw.exe` as the launcher to, theoretically, avoid +- `pywscript`: same as above, but uses `pythonw.exe` as the launcher to, theoretically, avoid launching a console window next to your application. -* `webbrowser`: alias to `PREFIX/python -m webbrowser -t URL`. +- `webbrowser`: alias to `PREFIX/python -m webbrowser -t URL`. On Linux the command is specified with `cmd`, expressed as a list of strings. On MacOS, `cmd` is also taken, but in this case it's expected to be a raw string. diff --git a/cep-0012.md b/cep-0012.md index fdc50f6c..86a58246 100644 --- a/cep-0012.md +++ b/cep-0012.md @@ -52,11 +52,11 @@ The schema of `run_exports.json` will mimic the `repodata.json` structure whenev Each `run_exports` metadata `dict` can contain the following fields; each field accepts a list of strings (conda-build specs). -- `weak` -- `strong` -- `weak_constrains` -- `strong_constrains` -- `noarch` +* `weak` +* `strong` +* `weak_constrains` +* `strong_constrains` +* `noarch` ```json { @@ -136,7 +136,6 @@ If a package does not define `run_exports`, the corresponding entry in `packages > See the validation schema draft in [`run_exports.schema.json`](https://github.com/conda/schemas/pull/25). - ## Backwards compatibility This is a new feature, so there is no backwards compatibility to worry about. @@ -147,9 +146,9 @@ We could maintain the status quo and ask downstream infrastructure to maintain t We also considered _adding_ the `run_exports` metadata to `repodata.json`, but this has a few shortcomings: -- It would require extending the `repodata` schema, currently not formally standardized. -- It would increase the size of the already heavy `repodata.json` files. -- (Typed) repodata parsers would need to be updated to handle the new field. +* It would require extending the `repodata` schema, currently not formally standardized. +* It would increase the size of the already heavy `repodata.json` files. +* (Typed) repodata parsers would need to be updated to handle the new field. Finally, we studied whether the `run_exports` metadata already present in the `channeldata.json` metadata would be enough. However, this metadata is only presented per version (not build), so it is insufficient and incomplete. Changing the `channeldata.json` schema to include per-build `run_exports` metadata would be a breaking change (e.g. `conda-build`'s `--use-channeldata` option). @@ -165,11 +164,10 @@ Only package building infrastructure (such as conda-forge bots) will need to fet ## References -- [Initial proposal in `conda-index`](https://github.com/conda/conda-index/issues/102) -- [`write_run_exports` implementation in `conda-build`](https://github.com/conda/conda-build/blob/9fd6279cf510d34008fd0423c9efe364302e7589/conda_build/build.py#L1508-L1517) -- [`run_exports` schema in `conda-build`](https://github.com/conda/conda-build/blob/9fd6279cf510d34008fd0423c9efe364302e7589/conda_build/utils.py#L132C1-L138) -- [`RunExports` struct in `rattler-build`](https://github.com/prefix-dev/rattler-build/blob/1ac730501651fd124a086ee1db92a67cd5b55429/src/metadata.rs#L53-L66) - +* [Initial proposal in `conda-index`](https://github.com/conda/conda-index/issues/102) +* [`write_run_exports` implementation in `conda-build`](https://github.com/conda/conda-build/blob/9fd6279cf510d34008fd0423c9efe364302e7589/conda_build/build.py#L1508-L1517) +* [`run_exports` schema in `conda-build`](https://github.com/conda/conda-build/blob/9fd6279cf510d34008fd0423c9efe364302e7589/conda_build/utils.py#L132C1-L138) +* [`RunExports` struct in `rattler-build`](https://github.com/prefix-dev/rattler-build/blob/1ac730501651fd124a086ee1db92a67cd5b55429/src/metadata.rs#L53-L66) ## Copyright diff --git a/cep-0013.md b/cep-0013.md index c3b228db..e42da05d 100644 --- a/cep-0013.md +++ b/cep-0013.md @@ -29,7 +29,7 @@ fast parsing and building of recipes. A discussion was started on what a new recipe spec could or should look like. The fragments of this discussion can be found here: -https://github.com/mamba-org/conda-specs/blob/master/proposed_specs/recipe.md + The reason for a new spec are: - Make it easier to parse ("pure yaml"). conda-build uses a mix of comments and @@ -51,11 +51,13 @@ The reason for a new spec are: - Selectors use a YAML dictionary with `if / then / else` (vs. comments in conda-build) and are only allowed in lists (dependencies, scripts, ...). The syntax looks like: + ```yaml - if: win then: this else: that # optional ``` + - for inline values, the Jinja ternary operator can be used, e.g. `number: ${{ 0 if linux else 100 }}` @@ -90,7 +92,7 @@ script: This is a valid YAML dictionary. Selector if statements are simple boolean expressions and follow Python syntax. The following selectors are all valid: -``` +```python win and arm64 (osx or linux) and aarch64 something == "test" diff --git a/cep-0014.md b/cep-0014.md index 3d42cc6c..d6cf45c3 100644 --- a/cep-0014.md +++ b/cep-0014.md @@ -28,7 +28,7 @@ and types in a single document and document them. This is part of a larger effor A discussion was started on what a new recipe spec could or should look like. The fragments of this discussion can be found here: -https://github.com/mamba-org/conda-specs/blob/7b53425caa11357487cba3fa9c7397744edb41f8/proposed_specs/recipe.md + The reasons for a new spec are: - Make it easier to parse ("pure yaml"). conda-build uses a mix of comments and @@ -45,9 +45,9 @@ The reasons for a new spec are: Outputs and top-level package are mutually exclusive, and outputs have exactly the same structure as top-level keys. If outputs exist, the top-level keys are 'merged' with the output keys (e.g. for the about section). -# Format +## Format -## Schema version +### Schema version The implicit version of the YAML schema for a recipe is an integer 1. To discern between the "old" format and the new format, we utilize the file name. @@ -65,7 +65,7 @@ To benefit from autocompletion, and other LSP features in editors, we can add a # yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/73cd2eed94c576213c5f25ab57adf6d8c83e792a/schema.json ``` -## Context section +### Context section The context section is a dictionary with key-value pairs that can be used for string interpolation. The right-hand side of the key-value pair is a scalar (bool, number or string). @@ -86,9 +86,7 @@ context: other_variable: test_${{ variable }} ``` - - -## Package section +### Package section Required for recipes without outputs section. This is the only required section (for single-output recipes). @@ -313,11 +311,11 @@ target_directory: path patches: [path] ``` -### Removed source definitions +#### Removed source definitions SVN and HG (mercury) source definitions are removed as they are not relevant anymore. -## Requirements section +### Requirements section ```yaml requirements: @@ -345,7 +343,7 @@ requirements: A `PackageSelector` in the recipe is currently defined as a string with up to two whitespaces, looking like this: -``` +```text # examples: @@ -362,7 +360,6 @@ python 3.9.* > For example, `conda` MatchSpecs allow specifying a channel, a build number (in square brackets) and many additional things. > We might homogenize these things later on. - #### `RunExports` section The different kind of run exports that can be specified are: @@ -465,7 +462,7 @@ The downstream test element renders to a `test_downstream.json` file that contai downstream: PackageSelector ``` -## Outputs section +### Outputs section conda-build has very hard to understand behavior with multiple outputs. We propose some drastic simplifications. Each output in the new format has the same keys as a "top-level" recipe. @@ -502,7 +499,7 @@ Before the build, the outputs are topologically sorted by their dependencies. Ea > [!NOTE] > A previous version contained a idea for a "cache-only" output. We've moved that to a future CEP. -### Aside: variant computation for multiple outputs +#### Aside: variant computation for multiple outputs Multiple outputs are treated like individual recipes after the merging of the nodes is completed. Therefore, each variant is computed for each output individually. @@ -556,7 +553,7 @@ about: # doc_source_url: url ``` -## Extra section +### Extra section ```yaml # a free form YAML dictionary @@ -566,7 +563,7 @@ extra: ## Example recipe -What follows is an example recipe, using the YAML syntax discussed in https://github.com/conda-incubator/ceps/pull/54 +What follows is an example recipe, using the YAML syntax discussed in ```yaml context: diff --git a/cep-0015.md b/cep-0015.md index f4ebc7de..88bc0f51 100644 --- a/cep-0015.md +++ b/cep-0015.md @@ -24,7 +24,7 @@ policy. A minimal `repodata.json` looks like this. -``` +```json {"info": {"subdir": "linux-64"}, "packages": {}, "packages.conda": {"some-package-filename.conda":{...}}, @@ -34,7 +34,7 @@ A minimal `repodata.json` looks like this. Add `base_url` to the `info` object. Increment `repodata_version`. -``` +```jsin {"info": {"subdir": "...", "base_url":"https://repo.anaconda.com/repo/main/linux-64/"}, "packages": {}, "packages.conda": {"some-package-filename.conda":{...}}, diff --git a/cep-0016.md b/cep-0016.md index 93911e60..c6277c28 100644 --- a/cep-0016.md +++ b/cep-0016.md @@ -10,15 +10,15 @@ Implementation -# Sharded Repodata +## Sharded Repodata We propose a new "repodata" format that can be sparsely fetched. That means, generally, smaller fetches (only fetch what you need) and faster updates of existing repodata (only fetch what has changed). -## Motivation +### Motivation The current repodata format is a JSON file that contains all the packages in a given channel. Unfortunately, that means it grows with the number of packages in the channel. This is a problem for large channels like conda-forge, which has over 150,000+ packages. It becomes very slow to fetch, parse and update the repodata. -## Design goals +### Design goals 1. **Speed**: Fetching repodata MUST be very fast. Both in the hot- and cold-cache case. 2. **Easy to update**: The channel MUST be very easy to update when new packages become available. @@ -28,9 +28,9 @@ The current repodata format is a JSON file that contains all the packages in a g 6. **Client-side cacheable**: If a user has a hot cache the user SHOULD only have to download small incremental changes. Preferably as little communication as possible with the server should be required to check freshness of the data. 7. **Bandwidth optimized**: Any data that is transferred SHOULD be as small as possible. -## Previous work +### Previous work -### JLAP +#### JLAP In a previously proposed CEP, [JLAP](https://github.com/conda-incubator/ceps/pull/20) was introduced. With JLAP only the changes to an initially downloaded `repodata.json` file have to be downloaded which means the user drastically saves on bandwidth which in turn makes fetching repodata much faster. @@ -41,7 +41,7 @@ JLAP also does not save anything with a cold cache because the initial repodata Finally, the implementation of JLAP is quite complex which makes it hard to adopt for implementers. -### ZSTD compression +#### ZSTD compression A notable improvement is compressing the `repodata.json` with `zst` and serving that file. In practice this yields a file that is 20% of the original size (20-30 Mb for large cases). Although this is still quite a big file it's substantially smaller. @@ -49,7 +49,7 @@ However, the file still contains all repodata in the channel. This means the fil Because the file is relatively big this means that often a large `max-age` is used for caching which means it takes more time to propagate new packages through the ecosystem. -## Proposal +### Proposal We propose a "sharded" repodata format. It works by splitting the repodata into multiple files (one per package name) and recursively fetching the "shards". @@ -60,7 +60,7 @@ Additionally an index file stores the mapping from package name to shard hash. Although not explicitly required the server SHOULD support HTTP/2 to reduce the overhead of doing a massive number of requests. -### Repodata shard index +#### Repodata shard index The shard index is a file that is stored under `/repodata_shards.msgpack.zst`. It is a zstandard compressed `msgpack` file that contains a mapping from package name to shard hash. @@ -94,7 +94,7 @@ For a large case (conda-forge linux-64) this file is 670kb at the time of writin We suggest serving the file with a short lived `Cache-Control` `max-age` header of 60 seconds to an hour but we leave it up to the channel administrator to set a value that works for that channel. -### Repodata shard +#### Repodata shard Individual shards are stored under the URL `.msgpack.zst`. Where the `sha256` is the lower-case hex representation of the bytes from the index and `shards_base_url` is defined in the shard index. @@ -171,36 +171,35 @@ Implementers SHOULD ignore unknown keys, this allows adding additional keys to t Although these files can become relatively large (100s of kilobytes) typically for a large case (conda-forge) these files remain very small, e.g. 100s of bytes to a couple of kilobytes. -## Fetch process +### Fetch process To fetch all needed package records, the client should implement the following steps: 1. Fetch the `repodata_shards.msgpack.zst` file. Standard HTTP caching semantics can be applied to this file. -3. For each package name, start fetching the corresponding hashes from the index file (for both arch & and noarch). +2. For each package name, start fetching the corresponding hashes from the index file (for both arch & and noarch). Shards can be cached locally and because they are content-addressable no additional round-trips to the server are required to check freshness. The server should also mark these with an `immutable` `Cache-Control` header. -4. Parsing the requirements of the fetched records and add the package names of the requirements to the set of packages to fetch. -5. Loop back to 1. until there are no new package names to fetch. +3. Parsing the requirements of the fetched records and add the package names of the requirements to the set of packages to fetch. +4. Loop back to 1. until there are no new package names to fetch. -## Garbage collection +### Garbage collection To avoid the cache from growing indefinitely, we propose to implement a garbage collection mechanism that removes shards that have no entry in the index file. The server should keep old shards for a certain amount of time (e.g. 1 week) to allow for clients with older shard-index data to fetch the previous versions. On the client side, a garbage collection process should run every so often to remove old shards from the cache. This can be done by comparing the cached shards with the index file and removing those that are not referenced anymore. +### Rejected ideas -## Rejected ideas - -### SHA hash compression +#### SHA hash compression SHA hashes are non-compressable because in the eyes of the compressor it is just random data. We have investigated using a binary prefix tree to enable better compression but this increased the complexity of the implementation quite a bit which conflicts with our goal of keeping things simple. -### Shorter SHA hashes +#### Shorter SHA hashes Another approach would be to only store the first 100 bytes or so of the SHA hash. This reduces the total size of the sha hashes significantly but it makes the client side implementation more complex because hash conflicts become an issue. This also makes a future implementation based on a OCI registry harder because layers in an OCI registry are also referenced by SHA256 hash. -### Storing the data as a struct of arrays +#### Storing the data as a struct of arrays To improve compression we investigated storing the index file as a struct of arrays instead of as an array of structs: @@ -228,27 +227,27 @@ vs This did yield slightly better compression but we felt it makes it slightly harder to implement and adapt in the future which we deemed not worth the small size decrease. -## Future improvements +### Future improvements -### Authentication +#### Authentication With this approach a client could do hundreds of requests concurrently. Authenticating all these requests would impose a non-negligable overhead to each requests and will also require more server resources. Initially a one-time authentication flow similar to OCI registry tokens was added to the CEP but this has been removed to reduce the scope of the CEP. Another CEP will be introduced to amend this CEP with a performant authentication flow. -### Remove redundant keys +#### Remove redundant keys `platform` and `arch` can be removed because these can be inferred from `subdir`. -### Integrating additional data +#### Integrating additional data With the total size of the repodata reduced it becomes feasible to add additional fields directly to the repodata records. Examples are: -- add `purl` as a list of strings (Package URLs to reference to original source of the package) (See: https://github.com/conda-incubator/ceps/pull/63) -- add `run_exports` as a list of strings (run-exports needed to build the package) (See: https://github.com/conda-incubator/ceps/pull/51) +- add `purl` as a list of strings (Package URLs to reference to original source of the package) (See: ) +- add `run_exports` as a list of strings (run-exports needed to build the package) (See: ) -### Update optimization +#### Update optimization We could implement support for smaller index update files. This can be done by creating a rolling daily and weekly index update file that can be used instead of fetching the whole `repodata_shards.msgpack.zst` file. The update operation is very simple (just update the hashmap with the new entries). @@ -259,6 +258,6 @@ For this we propose to add the following two files: They will contain the same format as the `repodata_shards.msgpack.zst` file but only contain the packages that have been updated in the last day or week respectively. `null` is used for keys that have been removed. The `created_at` field in the index file can be used to determine which file to fetch to make sure that the client has the latest information. -### Store `set(dependencies)` at the start of the shards or in a header +#### Store `set(dependencies)` at the start of the shards or in a header To reduce the time it takes to parse a shard and start fetching its dependencies we could also store the set of all dependencies in the file at the start of the shard or in a separate header. This could enable fetching recursive dependencies while still parsing the records. diff --git a/cep-0017.md b/cep-0017.md index d38d62a7..863a9393 100644 --- a/cep-0017.md +++ b/cep-0017.md @@ -67,7 +67,7 @@ def is_valid(target_prefix: str, python_site_packages_path: str) -> bool: When a package which includes this optional field is indexed the `python_site_packages_path` field will be included in the repodata entry for the package. For example the repodata entry for a `python-3.13.0rc1` package using the free-threading build configuration might look as follows: -``` +```json "python-3.13.0rc1-haa6bb3f_0_cpython_cp313t.tar.bz2": { "build": "haa6bb3f_0_cpython_cp313t", "build_number": 0, diff --git a/cep-0019.md b/cep-0019.md index ec0d6759..a19c8c11 100644 --- a/cep-0019.md +++ b/cep-0019.md @@ -23,15 +23,16 @@ Given a directory, propose an algorithm to compute the aggregated hash of its co Given a directory, recursively scan all its contents (without following symlinks) and sort them by their full path as a Unicode string. More specifically, it MUST follow an ascending lexicographical comparison using the numerical Unicode code points (i.e. the result of Python's built-in function `ord()`) of their characters [^1]. For each entry in the contents table, compute the hash for the concatenation of: + - UTF-8 encoded bytes of the path, relative to the input directory. Backslashes MUST be normalized to forward slashes before encoding. - Then, depending on the type: - - For regular files: - - If text, the UTF-8 encoded bytes of an `F` separator, followed by the UTF-8 encoded bytes of its line-ending-normalized contents (`\r\n` replaced with `\n`). A file is considered a text file if all the contents can be UTF-8 decoded. Otherwise it's considered binary. If the file can't be opened, it's handled as if it were empty. - - If binary, the UTF-8 encoded bytes of an `F` separator, followed by the bytes of its contents. - - If it can't be read, error out. - - For a directory, the UTF-8 encoded bytes of a `D` separator, and nothing else. - - For a symlink, the UTF-8 encoded bytes of an `L` separator, followed by the UTF-8 encoded bytes of the path it points to. Backslashes MUST be normalized to forward slashes before encoding. - - For any other types, error out. + - For regular files: + - If text, the UTF-8 encoded bytes of an `F` separator, followed by the UTF-8 encoded bytes of its line-ending-normalized contents (`\r\n` replaced with `\n`). A file is considered a text file if all the contents can be UTF-8 decoded. Otherwise it's considered binary. If the file can't be opened, it's handled as if it were empty. + - If binary, the UTF-8 encoded bytes of an `F` separator, followed by the bytes of its contents. + - If it can't be read, error out. + - For a directory, the UTF-8 encoded bytes of a `D` separator, and nothing else. + - For a symlink, the UTF-8 encoded bytes of an `L` separator, followed by the UTF-8 encoded bytes of the path it points to. Backslashes MUST be normalized to forward slashes before encoding. + - For any other types, error out. - UTF-8 encoded bytes of the string `-`. Note that the algorithm MUST error out on unreadable files and unknown file types because we can't verify its contents. An attacker could hide malicious content in those paths known to be "unhashable" and later reveal then again in the build script (e.g. by `chmod`ing them as readable). @@ -104,6 +105,7 @@ All CEPs are explicitly [CC0 1.0 Universal](https://creativecommons.org/publicdo +[RFC2119]: https://www.ietf.org/rfc/rfc2119.txt [fetchzip]: https://nixos.org/manual/nixpkgs/stable/#fetchurl [preimage]: https://flawed.net.nz/2018/02/21/attacking-merkle-trees-with-a-second-preimage-attack/ [dasher]: https://github.com/DrSLDR/dasher#hashing-scheme diff --git a/cep-0020.md b/cep-0020.md index 53c1bdf8..a926604e 100644 --- a/cep-0020.md +++ b/cep-0020.md @@ -82,6 +82,7 @@ A conda install tool does four things to support them: Entry points in `info/link.json` are materialized. ### info/link.json file + An example `info/link.json` for `noarch: python` looks like ```json @@ -108,7 +109,6 @@ An example for `info/link.json` for `noarch: generic` looks like } ``` - Here `preferred_env` is ignored by conda since 2017 and is not supported by other conda install tools. Therefore `info/link.json` is used exclusively for `noarch` packages and out of the two types, `noarch: generic` packages @@ -146,14 +146,15 @@ the package to the `noarch` subdir. Conda install tools have slightly different behavior. Conda: + 1. Actions `B1, B2, B3` are applied for packages with `A3`. 2. Action `B4` is applied for packages with `A4`. Micromamba: + 1. Actions `B1, B2, B3` are applied for packages with both `A2, A3`. 2. Action `B4` is applied for packages with both `A2, A4`. - ## Implementation for `abi3` packages ### Support in install tools @@ -210,7 +211,7 @@ requirements: ## Alternatives considered -### Apply all actions in a `post-link.sh` script. +### Apply all actions in a `post-link.sh` script A draft work provided at [python-feedstock][Python-pr-669] This was suggested by @mbargull, but some community members (@baszalmstra, @@ -219,7 +220,7 @@ code execution. However, in the author's opinion, this attack vector is not a new one since the install tool uses the Python executable in the host environment to compile the Python files. -### `noarch: python` packages with `__linux, __osx, __win` constrains. +### `noarch: python` packages with `__linux, __osx, __win` constrains This suggestion by `@wolfv` is not ideal as this clutters `noarch` subdir `repodata.json` file with packages that are useless for the platform in question. From 3d2196c38f73301446fcf4b98bdfafbac27def69 Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Fri, 14 Mar 2025 15:20:35 -0500 Subject: [PATCH 9/9] Update cep-0015.md --- cep-0015.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cep-0015.md b/cep-0015.md index 88bc0f51..845f3758 100644 --- a/cep-0015.md +++ b/cep-0015.md @@ -34,7 +34,7 @@ A minimal `repodata.json` looks like this. Add `base_url` to the `info` object. Increment `repodata_version`. -```jsin +```json {"info": {"subdir": "...", "base_url":"https://repo.anaconda.com/repo/main/linux-64/"}, "packages": {}, "packages.conda": {"some-package-filename.conda":{...}},