diff --git a/.chronus/changes/python-fix-azure-spec-detection-2026-3-29-12-59-39.md b/.chronus/changes/python-fix-azure-spec-detection-2026-3-29-12-59-39.md new file mode 100644 index 00000000000..0afd9ff9e7f --- /dev/null +++ b/.chronus/changes/python-fix-azure-spec-detection-2026-3-29-12-59-39.md @@ -0,0 +1,7 @@ +--- +changeKind: internal +packages: + - "@typespec/http-client-python" +--- + +Consolidate regenerate-common.ts into regenerate.ts and fix azure spec detection using azure-http-specs path check \ No newline at end of file diff --git a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts deleted file mode 100644 index 8dd426152b4..00000000000 --- a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts +++ /dev/null @@ -1,496 +0,0 @@ -import { promises } from "fs"; -import { dirname, join, relative, resolve } from "path"; - -// ---- Shared constants ---- - -export const SKIP_SPECS: string[] = ["type/file", "service/multiple-services"]; - -export const SpecialFlags: Record> = { - azure: { - "generate-test": true, - "generate-sample": true, - }, -}; - -// ---- Base emitter options (shared across repos) ---- - -export const BASE_AZURE_EMITTER_OPTIONS: Record< - string, - Record | Record[] -> = { - "azure/client-generator-core/access": { - namespace: "specs.azure.clientgenerator.core.access", - }, - "azure/client-generator-core/alternate-type": { - namespace: "specs.azure.clientgenerator.core.alternatetype", - }, - "azure/client-generator-core/api-version": { - namespace: "specs.azure.clientgenerator.core.apiversion", - }, - "azure/client-generator-core/client-initialization/default": { - namespace: "specs.azure.clientgenerator.core.clientinitialization.default", - }, - "azure/client-generator-core/client-initialization/individually": { - namespace: "specs.azure.clientgenerator.core.clientinitialization.individually", - }, - "azure/client-generator-core/client-initialization/individuallyParent": { - namespace: "specs.azure.clientgenerator.core.clientinitialization.individuallyparent", - }, - "azure/client-generator-core/client-location": { - namespace: "specs.azure.clientgenerator.core.clientlocation", - }, - "azure/client-generator-core/deserialize-empty-string-as-null": { - namespace: "specs.azure.clientgenerator.core.emptystring", - }, - "azure/client-generator-core/flatten-property": { - namespace: "specs.azure.clientgenerator.core.flattenproperty", - }, - "azure/client-generator-core/usage": { - namespace: "specs.azure.clientgenerator.core.usage", - }, - "azure/client-generator-core/client-doc": { - namespace: "specs.azure.clientgenerator.core.clientdoc", - }, - "azure/client-generator-core/override": { - namespace: "specs.azure.clientgenerator.core.override", - }, - "azure/client-generator-core/hierarchy-building": { - namespace: "specs.azure.clientgenerator.core.hierarchybuilding", - }, - "azure/core/basic": { - namespace: "specs.azure.core.basic", - }, - "azure/core/lro/rpc": { - namespace: "specs.azure.core.lro.rpc", - }, - "azure/core/lro/standard": { - namespace: "specs.azure.core.lro.standard", - }, - "azure/core/model": { - namespace: "specs.azure.core.model", - }, - "azure/core/page": { - namespace: "specs.azure.core.page", - }, - "azure/core/scalar": { - namespace: "specs.azure.core.scalar", - }, - "azure/core/traits": { - namespace: "specs.azure.core.traits", - }, - "azure/encode/duration": { - namespace: "specs.azure.encode.duration", - }, - "azure/example/basic": { - namespace: "specs.azure.example.basic", - }, - "azure/payload/pageable": { - namespace: "specs.azure.payload.pageable", - }, - "azure/versioning/previewVersion": { - namespace: "specs.azure.versioning.previewversion", - }, - "client/structure/default": { - namespace: "client.structure.service", - }, - "client/structure/multi-client": { - "package-name": "client-structure-multiclient", - namespace: "client.structure.multiclient", - }, - "client/structure/renamed-operation": { - "package-name": "client-structure-renamedoperation", - namespace: "client.structure.renamedoperation", - }, - "client/structure/two-operation-group": { - "package-name": "client-structure-twooperationgroup", - namespace: "client.structure.twooperationgroup", - }, - "client/naming": { - namespace: "client.naming.main", - }, - "client/overload": { - namespace: "client.overload", - }, - "encode/duration": { - namespace: "encode.duration", - }, - "encode/numeric": { - namespace: "encode.numeric", - }, - "parameters/basic": { - namespace: "parameters.basic", - }, - "parameters/spread": { - namespace: "parameters.spread", - }, - "payload/content-negotiation": { - namespace: "payload.contentnegotiation", - }, - "payload/multipart": { - namespace: "payload.multipart", - }, - "serialization/encoded-name/json": { - namespace: "serialization.encodedname.json", - }, - "special-words": { - namespace: "specialwords", - }, - "service/multi-service": { - namespace: "service.multiservice", - }, -}; - -export const BASE_EMITTER_OPTIONS: Record< - string, - Record | Record[] -> = { - "resiliency/srv-driven/old.tsp": { - "package-name": "resiliency-srv-driven1", - namespace: "resiliency.srv.driven1", - "package-mode": "azure-dataplane", - "package-pprint-name": "ResiliencySrvDriven1", - }, - "resiliency/srv-driven": { - "package-name": "resiliency-srv-driven2", - namespace: "resiliency.srv.driven2", - "package-mode": "azure-dataplane", - "package-pprint-name": "ResiliencySrvDriven2", - }, - "authentication/api-key": { - "clear-output-folder": "true", - }, - "authentication/http/custom": { - "package-name": "authentication-http-custom", - namespace: "authentication.http.custom", - "package-pprint-name": "Authentication Http Custom", - }, - "authentication/union": [ - { - "package-name": "authentication-union", - namespace: "authentication.union", - }, - { - "package-name": "setuppy-authentication-union", - namespace: "setuppy.authentication.union", - "keep-setup-py": "true", - }, - ], - "type/array": { - "package-name": "typetest-array", - namespace: "typetest.array", - }, - "type/dictionary": { - "package-name": "typetest-dictionary", - namespace: "typetest.dictionary", - }, - "type/enum/extensible": { - "package-name": "typetest-enum-extensible", - namespace: "typetest.enum.extensible", - }, - "type/enum/fixed": { - "package-name": "typetest-enum-fixed", - namespace: "typetest.enum.fixed", - }, - "type/model/empty": { - "package-name": "typetest-model-empty", - namespace: "typetest.model.empty", - }, - "type/model/inheritance/enum-discriminator": { - "package-name": "typetest-model-enumdiscriminator", - namespace: "typetest.model.enumdiscriminator", - }, - "type/model/inheritance/nested-discriminator": { - "package-name": "typetest-model-nesteddiscriminator", - namespace: "typetest.model.nesteddiscriminator", - }, - "type/model/inheritance/not-discriminated": { - "package-name": "typetest-model-notdiscriminated", - namespace: "typetest.model.notdiscriminated", - }, - "type/model/inheritance/single-discriminator": { - "package-name": "typetest-model-singlediscriminator", - namespace: "typetest.model.singlediscriminator", - }, - "type/model/inheritance/recursive": [ - { - "package-name": "typetest-model-recursive", - namespace: "typetest.model.recursive", - }, - { - // basic test for configuration "generation-subdir" - "package-name": "generation-subdir", - namespace: "generation.subdir", - "generation-subdir": "_generated", - "generate-test": "false", - "clear-output-folder": "true", - }, - ], - "type/model/usage": { - "package-name": "typetest-model-usage", - namespace: "typetest.model.usage", - }, - "type/model/visibility": [ - { - "package-name": "typetest-model-visibility", - namespace: "typetest.model.visibility", - }, - { - "package-name": "headasbooleantrue", - namespace: "headasbooleantrue", - "head-as-boolean": "true", - }, - { - "package-name": "headasbooleanfalse", - namespace: "headasbooleanfalse", - "head-as-boolean": "false", - }, - ], - "type/property/nullable": { - "package-name": "typetest-property-nullable", - namespace: "typetest.property.nullable", - }, - "type/property/optionality": { - "package-name": "typetest-property-optional", - namespace: "typetest.property.optional", - }, - "type/property/additional-properties": { - "package-name": "typetest-property-additionalproperties", - namespace: "typetest.property.additionalproperties", - }, - "type/scalar": { - "package-name": "typetest-scalar", - namespace: "typetest.scalar", - }, - "type/property/value-types": { - "package-name": "typetest-property-valuetypes", - namespace: "typetest.property.valuetypes", - }, - "type/union": { - "package-name": "typetest-union", - namespace: "typetest.union", - }, - "type/union/discriminated": { - "package-name": "typetest-discriminatedunion", - namespace: "typetest.discriminatedunion", - }, - "type/file": { - "package-name": "typetest-file", - namespace: "typetest.file", - }, - documentation: { - "package-name": "specs-documentation", - namespace: "specs.documentation", - }, - "versioning/added": [ - { - "package-name": "versioning-added", - namespace: "versioning.added", - }, - // check whether import of _validation.py/_types.py works when "generation-subdir" is configured - { - "package-name": "generation-subdir2", - namespace: "generation.subdir2", - "generate-test": "false", - "generation-subdir": "_generated", - }, - ], -}; - -// ---- Shared interfaces ---- - -export interface TspCommand { - outputDir: string; - command: string | string[]; -} - -export interface RegenerateFlagsInput { - flavor?: string; - debug?: boolean; - name?: string; - pyodide?: boolean; - jobs?: number; -} - -export interface RegenerateFlags { - flavor: string; - debug: boolean; - name?: string; - pyodide?: boolean; -} - -export interface RegenerateConfig { - azureHttpSpecs: string; - httpSpecs: string; - emitterOptions: Record | Record[]>; - azureEmitterOptions: Record | Record[]>; - preprocess: (flags: RegenerateFlagsInput) => Promise; - getCmdList: (spec: string, flags: RegenerateFlags) => TspCommand[]; - executeCommand: (cmd: TspCommand) => Promise; -} - -// ---- Shared utility functions ---- - -export function toPosix(dir: string): string { - return dir.replace(/\\/g, "/"); -} - -export function getEmitterOption( - spec: string, - flavor: string, - config: RegenerateConfig, -): Record[] { - const specDir = spec.includes("azure") ? config.azureHttpSpecs : config.httpSpecs; - const relativeSpec = toPosix(relative(specDir, spec)); - const key = relativeSpec.includes("resiliency/srv-driven/old.tsp") - ? relativeSpec - : dirname(relativeSpec); - const emitter_options = config.emitterOptions[key] || - (flavor === "azure" ? config.azureEmitterOptions[key] : [{}]) || [{}]; - return Array.isArray(emitter_options) ? emitter_options : [emitter_options]; -} - -export async function getSubdirectories( - baseDir: string, - flags: RegenerateFlags, -): Promise { - const subdirectories: string[] = []; - - async function searchDir(currentDir: string) { - const items = await promises.readdir(currentDir, { withFileTypes: true }); - - const promisesArray = items.map(async (item) => { - const subDirPath = join(currentDir, item.name); - if (item.isDirectory()) { - const mainTspPath = join(subDirPath, "main.tsp"); - const clientTspPath = join(subDirPath, "client.tsp"); - - const mainTspRelativePath = toPosix(relative(baseDir, mainTspPath)); - - if (SKIP_SPECS.some((skipSpec) => mainTspRelativePath.includes(skipSpec))) return; - - const hasMainTsp = await promises - .access(mainTspPath) - .then(() => true) - .catch(() => false); - const hasClientTsp = await promises - .access(clientTspPath) - .then(() => true) - .catch(() => false); - - if (mainTspRelativePath.toLowerCase().includes(flags.name || "")) { - if (mainTspRelativePath.includes("resiliency/srv-driven")) { - subdirectories.push(resolve(subDirPath, "old.tsp")); - } - if (hasClientTsp) { - subdirectories.push(resolve(subDirPath, "client.tsp")); - } else if (hasMainTsp) { - subdirectories.push(resolve(subDirPath, "main.tsp")); - } - } - - // Recursively search in the subdirectory - await searchDir(subDirPath); - } - }); - - await Promise.all(promisesArray); - } - - await searchDir(baseDir); - return subdirectories; -} - -export async function runTaskPool( - tasks: Array<() => Promise>, - poolLimit: number, -): Promise { - const executing: Set> = new Set(); - - for (const task of tasks) { - // Start the task and remove from set when done - const p: Promise = task().finally(() => executing.delete(p)); - executing.add(p); - - // If at capacity, wait for one to complete - if (executing.size >= poolLimit) { - await Promise.race(executing); - } - } - - // Wait for remaining tasks - await Promise.all(executing); -} - -export async function regenerate( - flags: RegenerateFlagsInput, - config: RegenerateConfig, -): Promise { - if (flags.flavor === undefined) { - await regenerate({ flavor: "azure", ...flags }, config); - await regenerate({ flavor: "unbranded", ...flags }, config); - } else { - await config.preprocess(flags); - - const flagsResolved: RegenerateFlags = { debug: false, flavor: flags.flavor, ...flags }; - const subdirectoriesForAzure = await getSubdirectories(config.azureHttpSpecs, flagsResolved); - const subdirectoriesForNonAzure = await getSubdirectories(config.httpSpecs, flagsResolved); - const subdirectories = - flags.flavor === "azure" - ? [...subdirectoriesForAzure, ...subdirectoriesForNonAzure] - : subdirectoriesForNonAzure; - const cmdList: TspCommand[] = subdirectories.flatMap((subdirectory) => - config.getCmdList(subdirectory, flagsResolved), - ); - - // Create tasks as functions for the pool - const tasks: Array<() => Promise> = cmdList.map((tspCommand) => { - return () => config.executeCommand(tspCommand); - }); - - // Run tasks with a concurrency limit - // Default: 30 jobs, or use provided value - const poolLimit = flags.jobs ?? 30; - await runTaskPool(tasks, poolLimit); - } -} - -// Preprocess: create files that should be deleted after regeneration (for testing) -export async function preprocess(flavor: string, generatedFolder: string): Promise { - if (flavor === "azure") { - // Use tests/generated// structure (same as output) - const testsGeneratedDir = resolve(generatedFolder, "../tests/generated/azure"); - - const DELETE_CONTENT = "# This file is to be deleted after regeneration"; - const DELETE_FILE = "to_be_deleted.py"; - const entries: { folder: string[]; file: string; content: string }[] = [ - { - folder: ["authentication-api-key", "authentication", "apikey", "_operations"], - file: DELETE_FILE, - content: DELETE_CONTENT, - }, - { - folder: ["generation-subdir", "generation", "subdir", "_generated"], - file: DELETE_FILE, - content: DELETE_CONTENT, - }, - { - folder: ["generation-subdir", "generated_tests"], - file: DELETE_FILE, - content: DELETE_CONTENT, - }, - { - folder: ["generation-subdir", "generation", "subdir"], - file: "to_be_kept.py", - content: "# This file is to be kept after regeneration", - }, - ]; - - await Promise.all( - entries.map(async ({ folder, file, content }) => { - const targetFolder = join(testsGeneratedDir, ...folder); - await promises.mkdir(targetFolder, { recursive: true }); - await promises.writeFile(join(targetFolder, file), content); - }), - ); - } -} diff --git a/packages/http-client-python/eng/scripts/ci/regenerate.ts b/packages/http-client-python/eng/scripts/ci/regenerate.ts index 958c871849e..a65230db578 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate.ts @@ -8,21 +8,13 @@ import { compile, NodeHost } from "@typespec/compiler"; import { execSync } from "child_process"; -import { existsSync, readdirSync, rmSync } from "fs"; +import { existsSync, rmSync } from "fs"; +import { access, mkdir, readdir, writeFile } from "fs/promises"; import { platform } from "os"; import { dirname, join, relative, resolve } from "path"; import pc from "picocolors"; import { fileURLToPath } from "url"; import { parseArgs } from "util"; -import { - BASE_AZURE_EMITTER_OPTIONS, - BASE_EMITTER_OPTIONS, - getSubdirectories, - preprocess, - SpecialFlags, - toPosix, - type RegenerateFlags, -} from "./regenerate-common.js"; // Parse arguments const argv = parseArgs({ @@ -83,21 +75,151 @@ ${pc.bold("Examples:")} process.exit(0); } -// Get paths -const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)); -const PLUGIN_DIR = argv.values.pluginDir - ? resolve(argv.values.pluginDir) - : resolve(SCRIPT_DIR, "../../../"); -const AZURE_HTTP_SPECS = resolve(PLUGIN_DIR, "node_modules/@azure-tools/azure-http-specs/specs"); -const HTTP_SPECS = resolve(PLUGIN_DIR, "node_modules/@typespec/http-specs/specs"); -const GENERATED_FOLDER = argv.values.generatedFolder - ? resolve(argv.values.generatedFolder) - : resolve(PLUGIN_DIR, "generator"); -const EMITTER_NAME = argv.values.emitterName || "@typespec/http-client-python"; +// ---- Shared constants ---- + +const SKIP_SPECS: string[] = ["type/file", "service/multiple-services"]; + +const SpecialFlags: Record> = { + azure: { + "generate-test": true, + "generate-sample": true, + }, +}; + +function toPosix(dir: string): string { + return dir.replace(/\\/g, "/"); +} + +interface RegenerateFlags { + flavor: string; + debug: boolean; + name?: string; + pyodide?: boolean; +} + +// ---- Base emitter options ---- -// Emitter options const AZURE_EMITTER_OPTIONS: Record | Record[]> = { - ...BASE_AZURE_EMITTER_OPTIONS, + "azure/client-generator-core/access": { + namespace: "specs.azure.clientgenerator.core.access", + }, + "azure/client-generator-core/alternate-type": { + namespace: "specs.azure.clientgenerator.core.alternatetype", + }, + "azure/client-generator-core/api-version": { + namespace: "specs.azure.clientgenerator.core.apiversion", + }, + "azure/client-generator-core/client-initialization/default": { + namespace: "specs.azure.clientgenerator.core.clientinitialization.default", + }, + "azure/client-generator-core/client-initialization/individually": { + namespace: "specs.azure.clientgenerator.core.clientinitialization.individually", + }, + "azure/client-generator-core/client-initialization/individuallyParent": { + namespace: "specs.azure.clientgenerator.core.clientinitialization.individuallyparent", + }, + "azure/client-generator-core/client-location": { + namespace: "specs.azure.clientgenerator.core.clientlocation", + }, + "azure/client-generator-core/deserialize-empty-string-as-null": { + namespace: "specs.azure.clientgenerator.core.emptystring", + }, + "azure/client-generator-core/flatten-property": { + namespace: "specs.azure.clientgenerator.core.flattenproperty", + }, + "azure/client-generator-core/usage": { + namespace: "specs.azure.clientgenerator.core.usage", + }, + "azure/client-generator-core/client-doc": { + namespace: "specs.azure.clientgenerator.core.clientdoc", + }, + "azure/client-generator-core/override": { + namespace: "specs.azure.clientgenerator.core.override", + }, + "azure/client-generator-core/hierarchy-building": { + namespace: "specs.azure.clientgenerator.core.hierarchybuilding", + }, + "azure/core/basic": { + namespace: "specs.azure.core.basic", + }, + "azure/core/lro/rpc": { + namespace: "specs.azure.core.lro.rpc", + }, + "azure/core/lro/standard": { + namespace: "specs.azure.core.lro.standard", + }, + "azure/core/model": { + namespace: "specs.azure.core.model", + }, + "azure/core/page": { + namespace: "specs.azure.core.page", + }, + "azure/core/scalar": { + namespace: "specs.azure.core.scalar", + }, + "azure/core/traits": { + namespace: "specs.azure.core.traits", + }, + "azure/encode/duration": { + namespace: "specs.azure.encode.duration", + }, + "azure/example/basic": { + namespace: "specs.azure.example.basic", + }, + "azure/payload/pageable": { + namespace: "specs.azure.payload.pageable", + }, + "azure/versioning/previewVersion": { + namespace: "specs.azure.versioning.previewversion", + }, + "client/structure/default": { + namespace: "client.structure.service", + }, + "client/structure/multi-client": { + "package-name": "client-structure-multiclient", + namespace: "client.structure.multiclient", + }, + "client/structure/renamed-operation": { + "package-name": "client-structure-renamedoperation", + namespace: "client.structure.renamedoperation", + }, + "client/structure/two-operation-group": { + "package-name": "client-structure-twooperationgroup", + namespace: "client.structure.twooperationgroup", + }, + "client/naming": { + namespace: "client.naming.main", + }, + "client/overload": { + namespace: "client.overload", + }, + "encode/duration": { + namespace: "encode.duration", + }, + "encode/numeric": { + namespace: "encode.numeric", + }, + "parameters/basic": { + namespace: "parameters.basic", + }, + "parameters/spread": { + namespace: "parameters.spread", + }, + "payload/content-negotiation": { + namespace: "payload.contentnegotiation", + }, + "payload/multipart": { + namespace: "payload.multipart", + }, + "serialization/encoded-name/json": { + namespace: "serialization.encodedname.json", + }, + "special-words": { + namespace: "specialwords", + }, + "service/multi-service": { + namespace: "service.multiservice", + }, "client/structure/client-operation-group": { "package-name": "client-structure-clientoperationgroup", namespace: "client.structure.clientoperationgroup", @@ -105,13 +227,254 @@ const AZURE_EMITTER_OPTIONS: Record | Record | Record[]> = { - ...BASE_EMITTER_OPTIONS, + "resiliency/srv-driven/old.tsp": { + "package-name": "resiliency-srv-driven1", + namespace: "resiliency.srv.driven1", + "package-mode": "azure-dataplane", + "package-pprint-name": "ResiliencySrvDriven1", + }, + "resiliency/srv-driven": { + "package-name": "resiliency-srv-driven2", + namespace: "resiliency.srv.driven2", + "package-mode": "azure-dataplane", + "package-pprint-name": "ResiliencySrvDriven2", + }, + "authentication/api-key": { + "clear-output-folder": "true", + }, + "authentication/http/custom": { + "package-name": "authentication-http-custom", + namespace: "authentication.http.custom", + "package-pprint-name": "Authentication Http Custom", + }, + "authentication/union": [ + { + "package-name": "authentication-union", + namespace: "authentication.union", + }, + { + "package-name": "setuppy-authentication-union", + namespace: "setuppy.authentication.union", + "keep-setup-py": "true", + }, + ], "type/array": { "package-name": "typetest-array", namespace: "typetest.array", }, + "type/dictionary": { + "package-name": "typetest-dictionary", + namespace: "typetest.dictionary", + }, + "type/enum/extensible": { + "package-name": "typetest-enum-extensible", + namespace: "typetest.enum.extensible", + }, + "type/enum/fixed": { + "package-name": "typetest-enum-fixed", + namespace: "typetest.enum.fixed", + }, + "type/model/empty": { + "package-name": "typetest-model-empty", + namespace: "typetest.model.empty", + }, + "type/model/inheritance/enum-discriminator": { + "package-name": "typetest-model-enumdiscriminator", + namespace: "typetest.model.enumdiscriminator", + }, + "type/model/inheritance/nested-discriminator": { + "package-name": "typetest-model-nesteddiscriminator", + namespace: "typetest.model.nesteddiscriminator", + }, + "type/model/inheritance/not-discriminated": { + "package-name": "typetest-model-notdiscriminated", + namespace: "typetest.model.notdiscriminated", + }, + "type/model/inheritance/single-discriminator": { + "package-name": "typetest-model-singlediscriminator", + namespace: "typetest.model.singlediscriminator", + }, + "type/model/inheritance/recursive": [ + { + "package-name": "typetest-model-recursive", + namespace: "typetest.model.recursive", + }, + { + "package-name": "generation-subdir", + namespace: "generation.subdir", + "generation-subdir": "_generated", + "generate-test": "false", + "clear-output-folder": "true", + }, + ], + "type/model/usage": { + "package-name": "typetest-model-usage", + namespace: "typetest.model.usage", + }, + "type/model/visibility": [ + { + "package-name": "typetest-model-visibility", + namespace: "typetest.model.visibility", + }, + { + "package-name": "headasbooleantrue", + namespace: "headasbooleantrue", + "head-as-boolean": "true", + }, + { + "package-name": "headasbooleanfalse", + namespace: "headasbooleanfalse", + "head-as-boolean": "false", + }, + ], + "type/property/nullable": { + "package-name": "typetest-property-nullable", + namespace: "typetest.property.nullable", + }, + "type/property/optionality": { + "package-name": "typetest-property-optional", + namespace: "typetest.property.optional", + }, + "type/property/additional-properties": { + "package-name": "typetest-property-additionalproperties", + namespace: "typetest.property.additionalproperties", + }, + "type/scalar": { + "package-name": "typetest-scalar", + namespace: "typetest.scalar", + }, + "type/property/value-types": { + "package-name": "typetest-property-valuetypes", + namespace: "typetest.property.valuetypes", + }, + "type/union": { + "package-name": "typetest-union", + namespace: "typetest.union", + }, + "type/union/discriminated": { + "package-name": "typetest-discriminatedunion", + namespace: "typetest.discriminatedunion", + }, + "type/file": { + "package-name": "typetest-file", + namespace: "typetest.file", + }, + documentation: { + "package-name": "specs-documentation", + namespace: "specs.documentation", + }, + "versioning/added": [ + { + "package-name": "versioning-added", + namespace: "versioning.added", + }, + { + "package-name": "generation-subdir2", + namespace: "generation.subdir2", + "generate-test": "false", + "generation-subdir": "_generated", + }, + ], }; +// ---- Shared utility functions ---- + +async function getSubdirectories(baseDir: string, flags: RegenerateFlags): Promise { + const subdirectories: string[] = []; + + async function searchDir(currentDir: string) { + const items = await readdir(currentDir, { withFileTypes: true }); + + const promisesArray = items.map(async (item) => { + const subDirPath = join(currentDir, item.name); + if (item.isDirectory()) { + const mainTspPath = join(subDirPath, "main.tsp"); + const clientTspPath = join(subDirPath, "client.tsp"); + + const mainTspRelativePath = toPosix(relative(baseDir, mainTspPath)); + + if (SKIP_SPECS.some((skipSpec) => mainTspRelativePath.includes(skipSpec))) return; + + const hasMainTsp = await access(mainTspPath) + .then(() => true) + .catch(() => false); + const hasClientTsp = await access(clientTspPath) + .then(() => true) + .catch(() => false); + + if (mainTspRelativePath.toLowerCase().includes(flags.name || "")) { + if (mainTspRelativePath.includes("resiliency/srv-driven")) { + subdirectories.push(resolve(subDirPath, "old.tsp")); + } + if (hasClientTsp) { + subdirectories.push(resolve(subDirPath, "client.tsp")); + } else if (hasMainTsp) { + subdirectories.push(resolve(subDirPath, "main.tsp")); + } + } + + await searchDir(subDirPath); + } + }); + + await Promise.all(promisesArray); + } + + await searchDir(baseDir); + return subdirectories; +} + +async function preprocess(flavor: string, generatedFolder: string): Promise { + if (flavor === "azure") { + const testsGeneratedDir = resolve(generatedFolder, "../tests/generated/azure"); + + const DELETE_CONTENT = "# This file is to be deleted after regeneration"; + const DELETE_FILE = "to_be_deleted.py"; + const entries: { folder: string[]; file: string; content: string }[] = [ + { + folder: ["authentication-api-key", "authentication", "apikey", "_operations"], + file: DELETE_FILE, + content: DELETE_CONTENT, + }, + { + folder: ["generation-subdir", "generation", "subdir", "_generated"], + file: DELETE_FILE, + content: DELETE_CONTENT, + }, + { + folder: ["generation-subdir", "generated_tests"], + file: DELETE_FILE, + content: DELETE_CONTENT, + }, + { + folder: ["generation-subdir", "generation", "subdir"], + file: "to_be_kept.py", + content: "# This file is to be kept after regeneration", + }, + ]; + + await Promise.all( + entries.map(async ({ folder, file, content }) => { + const targetFolder = join(testsGeneratedDir, ...folder); + await mkdir(targetFolder, { recursive: true }); + await writeFile(join(targetFolder, file), content); + }), + ); + } +} + +// Get paths +const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)); +const PLUGIN_DIR = argv.values.pluginDir + ? resolve(argv.values.pluginDir) + : resolve(SCRIPT_DIR, "../../../"); +const AZURE_HTTP_SPECS = resolve(PLUGIN_DIR, "node_modules/@azure-tools/azure-http-specs/specs"); +const HTTP_SPECS = resolve(PLUGIN_DIR, "node_modules/@typespec/http-specs/specs"); +const GENERATED_FOLDER = argv.values.generatedFolder + ? resolve(argv.values.generatedFolder) + : resolve(PLUGIN_DIR, "generator"); +const EMITTER_NAME = argv.values.emitterName || "@typespec/http-client-python"; + interface CompileTask { spec: string; outputDir: string; @@ -124,15 +487,22 @@ interface TaskGroup { tasks: CompileTask[]; } +// Check whether a spec path belongs to azure-http-specs (vs standard http-specs). +// Using "azure-http-specs" instead of "azure" to avoid false positives when the +// working directory path contains "azure" (e.g. azure-sdk-for-python). +function isAzureSpec(spec: string): boolean { + return spec.includes("azure-http-specs"); +} + function defaultPackageName(spec: string): string { - const specDir = spec.includes("azure") ? AZURE_HTTP_SPECS : HTTP_SPECS; + const specDir = isAzureSpec(spec) ? AZURE_HTTP_SPECS : HTTP_SPECS; return toPosix(relative(specDir, dirname(spec))) .replace(/\//g, "-") .toLowerCase(); } function getEmitterOptions(spec: string, flavor: string): Record[] { - const specDir = spec.includes("azure") ? AZURE_HTTP_SPECS : HTTP_SPECS; + const specDir = isAzureSpec(spec) ? AZURE_HTTP_SPECS : HTTP_SPECS; const relativeSpec = toPosix(relative(specDir, spec)); const key = relativeSpec.includes("resiliency/srv-driven/old.tsp") ? relativeSpec @@ -262,7 +632,7 @@ async function runParallel(groups: TaskGroup[], maxJobs: number): Promise { - const specDir = group.spec.includes("azure") ? AZURE_HTTP_SPECS : HTTP_SPECS; + const specDir = isAzureSpec(group.spec) ? AZURE_HTTP_SPECS : HTTP_SPECS; const shortName = toPosix(relative(specDir, dirname(group.spec))); // Run all tasks in this group sequentially to avoid state pollution @@ -311,16 +681,19 @@ async function runParallel(groups: TaskGroup[], maxJobs: number): Promise { const flavorDir = join(generatedDir, "..", "tests", "generated", flavor); - if (!existsSync(flavorDir)) return []; + try { + await access(flavorDir); + } catch { + return []; + } const configFiles: string[] = []; - for (const pkg of readdirSync(flavorDir, { withFileTypes: true })) { + for (const pkg of await readdir(flavorDir, { withFileTypes: true })) { if (pkg.isDirectory()) { const pkgDir = join(flavorDir, pkg.name); - // Find all .tsp-codegen-*.json files (supports multiple configs per output dir) - for (const file of readdirSync(pkgDir)) { + for (const file of await readdir(pkgDir)) { if (file.startsWith(".tsp-codegen-") && file.endsWith(".json")) { configFiles.push(join(pkgDir, file)); } @@ -412,7 +785,7 @@ async function regenerateFlavor( // Batch process all specs with Python const pyStartTime = performance.now(); - const configCount = collectConfigFiles(GENERATED_FOLDER, flavor).length; + const configCount = (await collectConfigFiles(GENERATED_FOLDER, flavor)).length; // Use fewer Python jobs since Python processing is heavier const pyJobs = Math.max(4, jobs); const pySuccess = runBatchPythonProcessing(flavor, configCount, pyJobs);