From cbecc7f34c7d5131da1b7182431500e4fe7a168b Mon Sep 17 00:00:00 2001 From: Matthew Lee Date: Wed, 25 Jun 2025 22:26:34 -0500 Subject: [PATCH 1/5] Support for python 3.9 --- .../src/language/Python/JSONPythonRenderer.ts | 4 +-- .../src/language/Python/PythonRenderer.ts | 25 +++++++++++++++++-- .../src/language/Python/constants.ts | 2 ++ .../src/language/Python/language.ts | 7 +++--- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts b/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts index b06965694..1e96cca0f 100644 --- a/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts +++ b/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts @@ -377,7 +377,7 @@ export class JSONPythonRenderer extends PythonRenderer { ", ", this.typingDecl("x", "Any"), ")", - this.typeHint(" -> ", this.withTyping("List"), "[", tvar, "]"), + this.typeHint(" -> ", this.pyOptions.features.builtinGenerics ? "list" : this.withTyping("List"), "[", tvar, "]"), ":", ], () => { @@ -618,7 +618,7 @@ export class JSONPythonRenderer extends PythonRenderer { (_integerType) => "int", (_doubleType) => "float", (_stringType) => "str", - (_arrayType) => "List", + (_arrayType) => this.pyOptions.features.builtinGenerics ? "list" : "List", (classType) => this.nameForNamedType(classType), (_mapType) => "dict", (enumType) => this.nameForNamedType(enumType), diff --git a/packages/quicktype-core/src/language/Python/PythonRenderer.ts b/packages/quicktype-core/src/language/Python/PythonRenderer.ts index fd7bb84e8..80592d2f2 100644 --- a/packages/quicktype-core/src/language/Python/PythonRenderer.ts +++ b/packages/quicktype-core/src/language/Python/PythonRenderer.ts @@ -162,14 +162,14 @@ export class PythonRenderer extends ConvenienceRenderer { (_doubletype) => "float", (_stringType) => "str", (arrayType) => [ - this.withTyping("List"), + this.pyOptions.features.builtinGenerics ? "list" : this.withTyping("List"), "[", this.pythonType(arrayType.items), "]", ], (classType) => this.namedType(classType), (mapType) => [ - this.withTyping("Dict"), + this.pyOptions.features.builtinGenerics ? "dict" : this.withTyping("Dict"), "[str, ", this.pythonType(mapType.values), "]", @@ -196,6 +196,14 @@ export class PythonRenderer extends ConvenienceRenderer { } if (nonNulls.size > 1) { + if (this.pyOptions.features.builtinGenerics) { + return [ + arrayIntercalate(" | ", memberTypes), + " | None", + ...rest, + ]; + } + this.withImport("typing", "Union"); return [ this.withTyping("Optional"), @@ -206,6 +214,13 @@ export class PythonRenderer extends ConvenienceRenderer { ]; } + if (this.pyOptions.features.builtinGenerics) { + return [ + defined(iterableFirst(memberTypes)), + " | None", + ...rest, + ]; + } return [ this.withTyping("Optional"), "[", @@ -215,6 +230,12 @@ export class PythonRenderer extends ConvenienceRenderer { ]; } + if (this.pyOptions.features.builtinGenerics) { + return [ + arrayIntercalate(" | ", memberTypes), + ]; + } + return [ this.withTyping("Union"), "[", diff --git a/packages/quicktype-core/src/language/Python/constants.ts b/packages/quicktype-core/src/language/Python/constants.ts index d32c10923..1c9660697 100644 --- a/packages/quicktype-core/src/language/Python/constants.ts +++ b/packages/quicktype-core/src/language/Python/constants.ts @@ -5,7 +5,9 @@ export const forbiddenTypeNames = [ "None", "Enum", "List", + "list", "Dict", + "dict", "Optional", "Union", "Iterable", diff --git a/packages/quicktype-core/src/language/Python/language.ts b/packages/quicktype-core/src/language/Python/language.ts index c6a35b0db..524435e0a 100644 --- a/packages/quicktype-core/src/language/Python/language.ts +++ b/packages/quicktype-core/src/language/Python/language.ts @@ -29,9 +29,10 @@ export const pythonOptions = { "python-version", "Python version", { - "3.5": { typeHints: false, dataClasses: false }, - "3.6": { typeHints: true, dataClasses: false }, - "3.7": { typeHints: true, dataClasses: true }, + "3.5": { typeHints: false, dataClasses: false, builtinGenerics: false }, + "3.6": { typeHints: true, dataClasses: false, builtinGenerics: false }, + "3.7": { typeHints: true, dataClasses: true, builtinGenerics: false }, + "3.9": { typeHints: true, dataClasses: true, builtinGenerics: true }, }, "3.6", ), From 2ad08bf4c4106e4b61cb3e7d377fd96b1894ba58 Mon Sep 17 00:00:00 2001 From: Matthew Lee Date: Thu, 26 Jun 2025 00:12:58 -0500 Subject: [PATCH 2/5] Fix bug --- .../quicktype-core/src/language/Python/JSONPythonRenderer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts b/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts index 1e96cca0f..04be1538a 100644 --- a/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts +++ b/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts @@ -429,7 +429,7 @@ export class JSONPythonRenderer extends PythonRenderer { ")", this.typeHint( " -> ", - this.withTyping("Dict"), + this.pyOptions.features.builtinGenerics ? "dict" : "Dict", "[str, ", tvar, "]", @@ -620,7 +620,7 @@ export class JSONPythonRenderer extends PythonRenderer { (_stringType) => "str", (_arrayType) => this.pyOptions.features.builtinGenerics ? "list" : "List", (classType) => this.nameForNamedType(classType), - (_mapType) => "dict", + (_mapType) => this.pyOptions.features.builtinGenerics ? "dict" : "Dict", (enumType) => this.nameForNamedType(enumType), (_unionType) => undefined, (transformedStringType) => { From 6b00ecab06859bccd7b5e448e0a94c997f02cf68 Mon Sep 17 00:00:00 2001 From: Matthew Lee Date: Thu, 26 Jun 2025 00:17:07 -0500 Subject: [PATCH 3/5] Update Python version in tests to 3.9 --- .github/workflows/test-pr.yaml | 388 ++++++++++++++++----------------- test/fixtures/python/run.sh | 2 +- 2 files changed, 195 insertions(+), 195 deletions(-) diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index d497d9b1c..5fc78c111 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -1,197 +1,197 @@ name: Test PR on: - pull_request: - branches: - - master - - "release/**" + pull_request: + branches: + - master + - "release/**" jobs: - build: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - uses: ./.github/workflows/setup - - test: - needs: [build] - runs-on: ${{ matrix.runs-on }} - - strategy: - fail-fast: true - - matrix: - fixture: - - typescript,typescript-zod,typescript-effect-schema - - javascript,schema-javascript - - golang,schema-golang - - cjson,schema-cjson - - cplusplus,schema-cplusplus - - flow,schema-flow - - java,schema-java - - python,schema-python - - haskell,schema-haskell - - csharp,schema-csharp,schema-json-csharp,graphql-csharp,csharp-SystemTextJson - - json-ts-csharp - - dart,schema-dart - # - swift,schema-swift # pgp issue - - javascript-prop-types - - ruby - - php - - scala3,schema-scala3 - - elixir,schema-elixir,graphql-elixir - - # Partially working - # - schema-typescript # TODO Unify with typescript once fixed - - # Implementation is too outdated to test in GitHub Actions - # - elm,schema-elm - - # Language is too niche / obscure to test easily on ubuntu-22.04 - # - pike,schema-pike - - # Not yet started - # @schani can you help me understand this fixture? - # It looks like it tests 13+ languages? - # - graphql - - # Never tested? - # - crystal - - runs-on: [ubuntu-22.04] - - include: - # Rust is very slow, so we use a larger runner - - fixture: rust,schema-rust - runs-on: ubuntu-latest-16-cores - # Kotlin is also slow - - fixture: kotlin,schema-kotlin,kotlin-jackson,schema-kotlin-jackson - runs-on: ubuntu-latest-16-cores - - # - fixture: objective-c # segfault on compiled test cmd - # runs-on: macos-latest - - name: ${{ matrix.fixture }} - steps: - - uses: actions/checkout@v3 - - uses: ./.github/workflows/setup - - - name: Setup PHP - if: ${{ contains(matrix.fixture, 'php') }} - uses: shivammathur/setup-php@v2 - with: - php-version: "8.2" - - - name: Setup Dart - if: ${{ contains(matrix.fixture, 'dart') }} - uses: dart-lang/setup-dart@v1.3 - with: - sdk: 2.14.4 - - - name: Setup Swift - if: ${{ contains(matrix.fixture, 'swift') }} - uses: swift-actions/setup-swift@v1 - with: - swift-version: "5.7.2" - - - name: Install Ruby - uses: ruby/setup-ruby@v1 - if: ${{ contains(matrix.fixture, 'ruby') }} - with: - ruby-version: "3.2.0" - - - name: Setup .NET Core SDK - if: ${{ contains(matrix.fixture, 'csharp') }} - uses: actions/setup-dotnet@v3 - with: - dotnet-version: 6 - - - name: Install Rust - if: ${{ contains(matrix.fixture, 'rust') }} - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - - name: Install Elm - if: ${{ contains(matrix.fixture, 'elm') }} - run: | - curl -L -o elm.gz https://github.com/elm/compiler/releases/download/0.19.1/binary-for-linux-64-bit.gz - gunzip elm.gz - chmod +x elm - sudo mv elm /usr/local/bin/ - - name: Install Haskell - if: ${{ contains(matrix.fixture, 'haskell') }} - run: | - if ! command -v stack > /dev/null 2>&1; then - curl -sL "https://get.haskellstack.org/" | sh - fi - - name: Install Python 3.7 - if: ${{ contains(matrix.fixture, 'python') }} - uses: actions/setup-python@v4 - with: - python-version: 3.7 - - - name: Install Python Dependencies - if: ${{ contains(matrix.fixture, 'python') }} - run: | - pip3.7 install mypy python-dateutil types-python-dateutil - - name: Install flow - if: ${{ contains(matrix.fixture, 'flow') }} - run: | - npm install -g flow-bin@0.66.0 flow-remove-types@1.2.3 - - name: Install Java - if: ${{ matrix.fixture == 'java,schema-java' || contains(matrix.fixture, 'kotlin') }} - uses: actions/setup-java@v3 - with: - java-version: "11" - distribution: "adopt" - - - name: Install Maven - if: ${{ matrix.fixture == 'java,schema-java' }} - uses: stCarolas/setup-maven@v4.5 - with: - maven-version: 3.8.2 - - - name: Install Kotlin - if: ${{ contains(matrix.fixture, 'kotlin') }} - uses: fwilhe2/setup-kotlin@main - - - name: Install go - if: ${{ contains(matrix.fixture, 'golang') }} - uses: actions/setup-go@v3 - with: - go-version: 1.15 - - - name: Install C - if: ${{ contains(matrix.fixture, 'cjson') }} - run: | - sudo apt-get update - sudo apt-get -y install build-essential valgrind - - - name: Install C++ - if: ${{ contains(matrix.fixture, 'cplusplus') }} - run: | - sudo apt-get update - sudo apt-get -y install libboost-all-dev software-properties-common g++ --assume-yes - - - name: Install scala - if: ${{ contains(matrix.fixture, 'scala3') }} - uses: VirtusLab/scala-cli-setup@main - - run: echo '@main def hello() = println("We need this spam print statement for bloop to exit correctly...")' | scala-cli _ - if: ${{ contains(matrix.fixture, 'scala3') }} - - - name: Install Elixir - if: ${{ contains(matrix.fixture, 'elixir') }} - uses: erlef/setup-beam@v1 - with: - elixir-version: "1.15.7" - otp-version: "26.0" - - - run: QUICKTEST=true FIXTURE=${{ matrix.fixture }} npm test - - test-complete: - if: ${{ cancelled() || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') }} - needs: test - runs-on: ubuntu-22.04 - steps: - - run: | - echo "Some workflows have failed!" - exit 1 + build: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - uses: ./.github/workflows/setup + + test: + needs: [build] + runs-on: ${{ matrix.runs-on }} + + strategy: + fail-fast: true + + matrix: + fixture: + - typescript,typescript-zod,typescript-effect-schema + - javascript,schema-javascript + - golang,schema-golang + - cjson,schema-cjson + - cplusplus,schema-cplusplus + - flow,schema-flow + - java,schema-java + - python,schema-python + - haskell,schema-haskell + - csharp,schema-csharp,schema-json-csharp,graphql-csharp,csharp-SystemTextJson + - json-ts-csharp + - dart,schema-dart + # - swift,schema-swift # pgp issue + - javascript-prop-types + - ruby + - php + - scala3,schema-scala3 + - elixir,schema-elixir,graphql-elixir + + # Partially working + # - schema-typescript # TODO Unify with typescript once fixed + + # Implementation is too outdated to test in GitHub Actions + # - elm,schema-elm + + # Language is too niche / obscure to test easily on ubuntu-22.04 + # - pike,schema-pike + + # Not yet started + # @schani can you help me understand this fixture? + # It looks like it tests 13+ languages? + # - graphql + + # Never tested? + # - crystal + + runs-on: [ubuntu-22.04] + + include: + # Rust is very slow, so we use a larger runner + - fixture: rust,schema-rust + runs-on: ubuntu-latest-16-cores + # Kotlin is also slow + - fixture: kotlin,schema-kotlin,kotlin-jackson,schema-kotlin-jackson + runs-on: ubuntu-latest-16-cores + + # - fixture: objective-c # segfault on compiled test cmd + # runs-on: macos-latest + + name: ${{ matrix.fixture }} + steps: + - uses: actions/checkout@v3 + - uses: ./.github/workflows/setup + + - name: Setup PHP + if: ${{ contains(matrix.fixture, 'php') }} + uses: shivammathur/setup-php@v2 + with: + php-version: "8.2" + + - name: Setup Dart + if: ${{ contains(matrix.fixture, 'dart') }} + uses: dart-lang/setup-dart@v1.3 + with: + sdk: 2.14.4 + + - name: Setup Swift + if: ${{ contains(matrix.fixture, 'swift') }} + uses: swift-actions/setup-swift@v1 + with: + swift-version: "5.7.2" + + - name: Install Ruby + uses: ruby/setup-ruby@v1 + if: ${{ contains(matrix.fixture, 'ruby') }} + with: + ruby-version: "3.2.0" + + - name: Setup .NET Core SDK + if: ${{ contains(matrix.fixture, 'csharp') }} + uses: actions/setup-dotnet@v3 + with: + dotnet-version: 6 + + - name: Install Rust + if: ${{ contains(matrix.fixture, 'rust') }} + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + + - name: Install Elm + if: ${{ contains(matrix.fixture, 'elm') }} + run: | + curl -L -o elm.gz https://github.com/elm/compiler/releases/download/0.19.1/binary-for-linux-64-bit.gz + gunzip elm.gz + chmod +x elm + sudo mv elm /usr/local/bin/ + - name: Install Haskell + if: ${{ contains(matrix.fixture, 'haskell') }} + run: | + if ! command -v stack > /dev/null 2>&1; then + curl -sL "https://get.haskellstack.org/" | sh + fi + - name: Install Python 3.9 + if: ${{ contains(matrix.fixture, 'python') }} + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install Python Dependencies + if: ${{ contains(matrix.fixture, 'python') }} + run: | + pip3.9 install mypy python-dateutil types-python-dateutil + - name: Install flow + if: ${{ contains(matrix.fixture, 'flow') }} + run: | + npm install -g flow-bin@0.66.0 flow-remove-types@1.2.3 + - name: Install Java + if: ${{ matrix.fixture == 'java,schema-java' || contains(matrix.fixture, 'kotlin') }} + uses: actions/setup-java@v3 + with: + java-version: "11" + distribution: "adopt" + + - name: Install Maven + if: ${{ matrix.fixture == 'java,schema-java' }} + uses: stCarolas/setup-maven@v4.5 + with: + maven-version: 3.8.2 + + - name: Install Kotlin + if: ${{ contains(matrix.fixture, 'kotlin') }} + uses: fwilhe2/setup-kotlin@main + + - name: Install go + if: ${{ contains(matrix.fixture, 'golang') }} + uses: actions/setup-go@v3 + with: + go-version: 1.15 + + - name: Install C + if: ${{ contains(matrix.fixture, 'cjson') }} + run: | + sudo apt-get update + sudo apt-get -y install build-essential valgrind + + - name: Install C++ + if: ${{ contains(matrix.fixture, 'cplusplus') }} + run: | + sudo apt-get update + sudo apt-get -y install libboost-all-dev software-properties-common g++ --assume-yes + + - name: Install scala + if: ${{ contains(matrix.fixture, 'scala3') }} + uses: VirtusLab/scala-cli-setup@main + - run: echo '@main def hello() = println("We need this spam print statement for bloop to exit correctly...")' | scala-cli _ + if: ${{ contains(matrix.fixture, 'scala3') }} + + - name: Install Elixir + if: ${{ contains(matrix.fixture, 'elixir') }} + uses: erlef/setup-beam@v1 + with: + elixir-version: "1.15.7" + otp-version: "26.0" + + - run: QUICKTEST=true FIXTURE=${{ matrix.fixture }} npm test + + test-complete: + if: ${{ cancelled() || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') }} + needs: test + runs-on: ubuntu-22.04 + steps: + - run: | + echo "Some workflows have failed!" + exit 1 diff --git a/test/fixtures/python/run.sh b/test/fixtures/python/run.sh index 0ad6e6787..98c4273a0 100755 --- a/test/fixtures/python/run.sh +++ b/test/fixtures/python/run.sh @@ -3,7 +3,7 @@ if [ "x$QUICKTYPE_PYTHON_VERSION" = "x2.7" ] ; then PYTHON="python2.7" else - PYTHON="python3.7" + PYTHON="python3.9" fi "$PYTHON" "$@" From 9824c3e46aa0b96ca40b27ee8c5d4fc06fc0251d Mon Sep 17 00:00:00 2001 From: Matthew Lee Date: Thu, 26 Jun 2025 02:21:42 -0500 Subject: [PATCH 4/5] Fixed bug --- .../src/language/Python/PythonRenderer.ts | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/quicktype-core/src/language/Python/PythonRenderer.ts b/packages/quicktype-core/src/language/Python/PythonRenderer.ts index 80592d2f2..f574a6dc1 100644 --- a/packages/quicktype-core/src/language/Python/PythonRenderer.ts +++ b/packages/quicktype-core/src/language/Python/PythonRenderer.ts @@ -144,13 +144,13 @@ export class PythonRenderer extends ConvenienceRenderer { return this.withImport("typing", name); } - protected namedType(t: Type): Sourcelike { + protected namedType(t: Type, _suppressQuotes: boolean = false): Sourcelike { const name = this.nameForNamedType(t); - if (this.declaredTypes.has(t)) return name; + if (_suppressQuotes || this.declaredTypes.has(t)) return name; return ["'", name, "'"]; } - protected pythonType(t: Type, _isRootTypeDef = false): Sourcelike { + protected pythonType(t: Type, _isRootTypeDef = false, _suppressQuotes = false): Sourcelike { const actualType = followTargetType(t); return matchType( @@ -167,20 +167,27 @@ export class PythonRenderer extends ConvenienceRenderer { this.pythonType(arrayType.items), "]", ], - (classType) => this.namedType(classType), + (classType) => this.namedType(classType, _suppressQuotes), (mapType) => [ this.pyOptions.features.builtinGenerics ? "dict" : this.withTyping("Dict"), "[str, ", this.pythonType(mapType.values), "]", ], - (enumType) => this.namedType(enumType), + (enumType) => this.namedType(enumType, _suppressQuotes), (unionType) => { const [hasNull, nonNulls] = removeNullFromUnion(unionType); + const hasClassOrEnumType = Array.from(nonNulls).some( + t => t instanceof ClassType || t instanceof EnumType + ); + const encapsulator = hasClassOrEnumType ? "'" : "" + const memberTypes = Array.from(nonNulls).map((m) => - this.pythonType(m), + // Suppress quotes for namedType + this.pythonType(m, false, true), ); + if (hasNull !== null) { const rest: string[] = []; if ( @@ -198,9 +205,11 @@ export class PythonRenderer extends ConvenienceRenderer { if (nonNulls.size > 1) { if (this.pyOptions.features.builtinGenerics) { return [ + encapsulator, arrayIntercalate(" | ", memberTypes), " | None", ...rest, + encapsulator ]; } @@ -216,9 +225,11 @@ export class PythonRenderer extends ConvenienceRenderer { if (this.pyOptions.features.builtinGenerics) { return [ + encapsulator, defined(iterableFirst(memberTypes)), " | None", ...rest, + encapsulator ]; } return [ @@ -232,7 +243,9 @@ export class PythonRenderer extends ConvenienceRenderer { if (this.pyOptions.features.builtinGenerics) { return [ + encapsulator, arrayIntercalate(" | ", memberTypes), + encapsulator ]; } From aba2303c959ef32b84631cb4ca8b8258ec596bc6 Mon Sep 17 00:00:00 2001 From: Matthew Lee Date: Thu, 26 Jun 2025 02:22:27 -0500 Subject: [PATCH 5/5] Use python 3.9 in tests --- packages/quicktype-core/src/language/Python/language.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/quicktype-core/src/language/Python/language.ts b/packages/quicktype-core/src/language/Python/language.ts index 524435e0a..6bac0bb73 100644 --- a/packages/quicktype-core/src/language/Python/language.ts +++ b/packages/quicktype-core/src/language/Python/language.ts @@ -34,7 +34,7 @@ export const pythonOptions = { "3.7": { typeHints: true, dataClasses: true, builtinGenerics: false }, "3.9": { typeHints: true, dataClasses: true, builtinGenerics: true }, }, - "3.6", + "3.9", ), justTypes: new BooleanOption("just-types", "Classes only", false), nicePropertyNames: new BooleanOption(