Add MCP conformance test coverage (#435)

<!-- Provide a brief summary of your changes -->

## Motivation and Context

1. The currently failing checks are commented out in `ConformanceTest`,
see the TODOs.
2. I added detailed logging for Gradle task, Server, and Client to allow
easier output reading when viewing tests results outside of IDE.
3. As a future improvement, instead of hard-coding scenarios, we can get
them with `npx @modelcontextprotocol/conformance list` and run all of
them.
4. The conformance framework writes its result into
`kotlin-sdk-test/results` directory and contains JSONs like the one
below:
```json
[
  {
    "id": "mcp-client-initialization",
    "name": "MCPClientInitialization",
    "description": "Validates that MCP client properly initializes with server",
    "status": "SUCCESS",
    "timestamp": "2025-11-28T14:38:36.718Z",
    "specReferences": [
      {
        "id": "MCP-Lifecycle",
        "url": "https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle"
      }
    ],
    "details": {
      "protocolVersionSent": "2025-06-18",
      "expectedSpecVersion": "2025-06-18",
      "versionMatch": true,
      "clientName": "kotlin-conformance-client",
      "clientVersion": "1.0.0"
    }
  },
  {
    "id": "server-info",
    "name": "ServerInfo",
    "description": "Test server info returned to client",
    "status": "INFO",
    "timestamp": "2025-11-28T14:38:36.718Z",
    "specReferences": [
      {
        "id": "MCP-Lifecycle",
        "url": "https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle"
      }
    ],
    "details": {
      "serverName": "test-server",
      "serverVersion": "1.0.0"
    }
  }
]
```
As another future improvement, we can do somethig with these JSON files,
like putting them into a job summary on GitHub.

5. The client test also generated stderr.txt file in
`kotlin-sdk-test/results/initialize-.......` which contains errors
because it tries to call a tool. I haven't investigated if it's just the
way I wrote the client or related to `StreamableHttpClientTransport`.
6. The server is the simplified version of the existing server for
Kotlin<--->Typescript tests. If conformance tests extend significantly,
we can either combine the server implementations or ditch the
integration tests.

## How Has This Been Tested?
<!-- Have you tested this in a real application? Which scenarios were
tested? -->

## Breaking Changes
<!-- Will users need to update their code or configurations? -->

## Types of changes
<!-- What types of changes does your code introduce? Put an `x` in all
the boxes that apply: -->
- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
- [ ] Documentation update

## Checklist
<!-- Go over all the following points, and put an `x` in all the boxes
that apply. -->
- [x] I have read the [MCP
Documentation](https://modelcontextprotocol.io)
- [x] My code follows the repository's style guidelines
- [ ] New and existing tests pass locally
- [ ] I have added appropriate error handling
- [ ] I have added or updated documentation as needed

## Additional context
<!-- Add any other context, implementation notes, or design decisions
-->

---------

Co-authored-by: Konstantin Pavlov <[email protected]>

Author: skarpovdev

.github/workflows/build.yml

@@ -41,6 +41,7 @@ jobs:
             clean \
             ktlintCheck \
             build \
+            -x :conformance-test:test \
             koverLog koverHtmlReport \
             publishToMavenLocal \
             -Pversion=0.0.1-SNAPSHOT

.github/workflows/conformance.yml

@@ -0,0 +1,50 @@
+name: Conformance Tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [ main ]
+  push:
+    branches: [ main ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  # Cancel only when the run is NOT on `main` branch
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main'  }}
+
+jobs:
+  run-conformance:
+    runs-on: macos-latest-xlarge
+    name: Run Conformance Tests
+    timeout-minutes: 20
+    env:
+      JAVA_OPTS: "-Xmx8g -Dfile.encoding=UTF-8 -Djava.awt.headless=true -Dkotlin.daemon.jvm.options=-Xmx6g"
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up JDK 21
+        uses: actions/setup-java@v5
+        with:
+          java-version: '21'
+          distribution: 'temurin'
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '24.11.1'
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v5
+        with:
+          add-job-summary: 'always'
+          cache-read-only: true
+
+      - name: Run Conformance Tests
+        run: ./gradlew --no-daemon :conformance-test:test
+
+      - name: Upload Conformance Results
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v5
+        with:
+          name: conformance-results
+          path: conformance-test/results/

.github/workflows/gradle-publish.yml

@@ -34,7 +34,7 @@ jobs:
         uses: gradle/actions/setup-gradle@v5
 
       - name: Clean Build with Gradle
-        run: ./gradlew clean build
+        run: ./gradlew clean build -x :conformance-test:test
 
       - name: Publish to Maven Central Portal
         id: publish

.gitignore

@@ -56,3 +56,6 @@ dist
 ### SWE agents ###
 .claude/
 .junie/
+
+### Conformance test results ###
+conformance-test/results/

conformance-test/build.gradle.kts

@@ -0,0 +1,38 @@
+import org.gradle.api.tasks.testing.logging.TestExceptionFormat
+
+plugins {
+    kotlin("jvm")
+}
+
+dependencies {
+    testImplementation(project(":kotlin-sdk"))
+    testImplementation(kotlin("test"))
+    testImplementation(libs.kotlin.logging)
+    testImplementation(libs.ktor.client.cio)
+    testImplementation(libs.ktor.server.cio)
+    testImplementation(libs.ktor.server.websockets)
+    testRuntimeOnly(libs.slf4j.simple)
+}
+
+tasks.test {
+    useJUnitPlatform()
+
+    testLogging {
+        events("passed", "skipped", "failed")
+        showStandardStreams = true
+        showExceptions = true
+        showCauses = true
+        showStackTraces = true
+        exceptionFormat = TestExceptionFormat.FULL
+    }
+
+    doFirst {
+        systemProperty("test.classpath", classpath.asPath)
+
+        println("\n" + "=".repeat(60))
+        println("MCP CONFORMANCE TESTS")
+        println("=".repeat(60))
+        println("These tests validate compliance with the MCP specification.")
+        println("=".repeat(60) + "\n")
+    }
+}

conformance-test/src/test/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt

@@ -0,0 +1,85 @@
+package io.modelcontextprotocol.kotlin.sdk.conformance
+
+import io.github.oshai.kotlinlogging.KotlinLogging
+import io.ktor.client.HttpClient
+import io.ktor.client.engine.cio.CIO
+import io.ktor.client.plugins.sse.SSE
+import io.modelcontextprotocol.kotlin.sdk.client.Client
+import io.modelcontextprotocol.kotlin.sdk.client.StreamableHttpClientTransport
+import io.modelcontextprotocol.kotlin.sdk.shared.Transport
+import io.modelcontextprotocol.kotlin.sdk.types.CallToolRequest
+import io.modelcontextprotocol.kotlin.sdk.types.CallToolRequestParams
+import io.modelcontextprotocol.kotlin.sdk.types.Implementation
+import kotlinx.coroutines.runBlocking
+import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.buildJsonObject
+
+private val logger = KotlinLogging.logger {}
+
+fun main(args: Array<String>) {
+    require(args.isNotEmpty()) {
+        "Server URL must be provided as an argument"
+    }
+
+    val serverUrl = args.last()
+    logger.info { "Connecting to test server at: $serverUrl" }
+
+    val httpClient = HttpClient(CIO) {
+        install(SSE)
+    }
+    val transport: Transport = StreamableHttpClientTransport(httpClient, serverUrl)
+
+    val client = Client(
+        clientInfo = Implementation(
+            name = "kotlin-conformance-client",
+            version = "1.0.0",
+        ),
+    )
+
+    var exitCode = 0
+
+    runBlocking {
+        try {
+            client.connect(transport)
+            logger.info { "✅ Connected to server successfully" }
+
+            try {
+                val tools = client.listTools()
+                logger.info { "Available tools: ${tools.tools.map { it.name }}" }
+
+                if (tools.tools.isNotEmpty()) {
+                    val toolName = tools.tools.first().name
+                    logger.info { "Calling tool: $toolName" }
+
+                    val result = client.callTool(
+                        CallToolRequest(
+                            params = CallToolRequestParams(
+                                name = toolName,
+                                arguments = buildJsonObject {
+                                    put("input", JsonPrimitive("test"))
+                                },
+                            ),
+                        ),
+                    )
+                    logger.info { "Tool result: ${result.content}" }
+                }
+            } catch (e: Exception) {
+                logger.debug(e) { "Error during tool operations (may be expected for some scenarios)" }
+            }
+
+            logger.info { "✅ Client operations completed successfully" }
+        } catch (e: Exception) {
+            logger.error(e) { "❌ Client failed" }
+            exitCode = 1
+        } finally {
+            try {
+                transport.close()
+            } catch (e: Exception) {
+                logger.warn(e) { "Error closing transport" }
+            }
+            httpClient.close()
+        }
+    }
+
+    kotlin.system.exitProcess(exitCode)
+}