chore: initial commit

johannschopplich · johannschopplich · commit 405614064067 · 2025-11-01T22:00:41.000+01:00
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,58 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.DS_Store
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+.nox/
+
+# Type checking
+.mypy_cache/
+.pytype/
+.pyre/
+.pyright/
+
+# uv
+.uv/
+uv.lock
diff --git a/.python-version b/.python-version
@@ -0,0 +1 @@
+3.14
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025-PRESENT Johann Schopplich
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
@@ -0,0 +1,57 @@
+# TOON Format for Python
+
+[![PyPI version](https://img.shields.io/pypi/v/toon-format.svg)](https://pypi.org/project/toon-format/)
+[![Python versions](https://img.shields.io/pypi/pyversions/toon-format.svg)](https://pypi.org/project/toon-format/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE)
+
+**Token-Oriented Object Notation** is a compact, human-readable format designed for passing structured data to Large Language Models with significantly reduced token usage.
+
+## Status
+
+🚧 **This package is currently a namespace reservation.** Full implementation coming soon!
+
+### Example
+
+**JSON** (verbose):
+```json
+{
+  "users": [
+    { "id": 1, "name": "Alice", "role": "admin" },
+    { "id": 2, "name": "Bob", "role": "user" }
+  ]
+}
+```
+
+**TOON** (compact):
+```
+users[2]{id,name,role}:
+  1,Alice,admin
+  2,Bob,user
+```
+
+## Resources
+
+- [TOON Specification](https://github.com/johannschopplich/toon/blob/main/SPEC.md)
+- [Main Repository](https://github.com/johannschopplich/toon)
+- [Benchmarks & Performance](https://github.com/johannschopplich/toon#benchmarks)
+- [Other Language Implementations](https://github.com/johannschopplich/toon#other-implementations)
+
+## Future Usage
+
+Once implemented, the package will provide:
+
+```python
+from toon_format import encode, decode
+
+data = # your data structure
+toon_string = encode(data)
+decoded = decode(toon_string)
+```
+
+## Contributing
+
+Interested in implementing TOON for Python? Check out the [specification](https://github.com/johannschopplich/toon/blob/main/SPEC.md) and feel free to contribute!
+
+## License
+
+MIT License © 2025-PRESENT [Johann Schopplich](https://github.com/johannschopplich)
diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,42 @@
+[project]
+name = "toon-format"
+version = "0.1.0"
+description = "Token-Oriented Object Notation – a token-efficient JSON alternative for LLM prompts"
+readme = "README.md"
+authors = [
+    { name = "Johann Schopplich", email = "hello@johannschopplich.com" }
+]
+requires-python = ">=3.10"
+dependencies = []
+license = { text = "MIT" }
+keywords = ["toon", "serialization", "llm", "data-format", "token-efficient"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+[project.urls]
+Homepage = "https://toonformat.dev"
+Repository = "https://github.com/toon-format/toon-python"
+Documentation = "https://github.com/toon-format/toon"
+"Bug Tracker" = "https://github.com/toon-format/toon-python/issues"
+
+[dependency-groups]
+dev = ["pytest>=8.0.0"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_functions = ["test_*"]
+
+[build-system]
+requires = ["uv_build>=0.9.7,<0.10.0"]
+build-backend = "uv_build"
diff --git a/src/toon_format/__init__.py b/src/toon_format/__init__.py
@@ -0,0 +1,13 @@
+"""
+Token-Oriented Object Notation (TOON) for Python.
+
+A compact, human-readable format designed for passing structured data
+to Large Language Models with significantly reduced token usage.
+"""
+
+from toon_format.decoder import decode
+from toon_format.encoder import encode
+from toon_format.types import DecodeOptions, EncodeOptions
+
+__version__ = "0.1.0"
+__all__ = ["encode", "decode", "EncodeOptions", "DecodeOptions"]
diff --git a/src/toon_format/decoder.py b/src/toon_format/decoder.py
@@ -0,0 +1,31 @@
+"""TOON decoder implementation."""
+
+from toon_format.types import DecodeOptions, JsonValue
+
+
+def decode(input: str, options: DecodeOptions | None = None) -> JsonValue:
+    """Convert a TOON-formatted string to a Python value.
+
+    Args:
+        input: A TOON-formatted string to parse
+        options: Optional decoding options:
+            - indent: Expected number of spaces per indentation level (default: 2)
+            - strict: Enable strict validation (default: True)
+
+    Returns:
+        A Python value (dict, list, or primitive) representing the parsed TOON data.
+
+    Raises:
+        ValueError: If the input is malformed (when strict=True)
+
+    Examples:
+        >>> decode('items[2]{sku,qty}:\\n  A1,2\\n  B2,1')
+        {'items': [{'sku': 'A1', 'qty': 2}, {'sku': 'B2', 'qty': 1}]}
+
+        >>> decode('tags[2]: foo,bar')
+        {'tags': ['foo', 'bar']}
+
+        >>> decode('[3]: 1,2,3')
+        [1, 2, 3]
+    """
+    raise NotImplementedError("TOON decoder is not yet implemented")
diff --git a/src/toon_format/encoder.py b/src/toon_format/encoder.py
@@ -0,0 +1,34 @@
+"""TOON encoder implementation."""
+
+from typing import Any
+
+from toon_format.types import EncodeOptions
+
+
+def encode(value: Any, options: EncodeOptions | None = None) -> str:
+    """Convert a value to TOON format.
+
+    Args:
+        value: Any JSON-serializable value (object, array, primitive, or nested structure).
+               Non-JSON-serializable values (functions, undefined, non-finite numbers) are
+               converted to null. Dates are converted to ISO strings, and BigInts are emitted
+               as decimal integers.
+        options: Optional encoding options:
+            - indent: Number of spaces per indentation level (default: 2)
+            - delimiter: Delimiter for array values and tabular rows (default: ',')
+            - length_marker: Optional marker to prefix array lengths (default: False)
+
+    Returns:
+        A TOON-formatted string with no trailing newline or spaces.
+
+    Examples:
+        >>> encode({"items": [{"sku": "A1", "qty": 2}, {"sku": "B2", "qty": 1}]})
+        'items[2]{sku,qty}:\\n  A1,2\\n  B2,1'
+
+        >>> encode({"tags": ["foo", "bar"]}, {"delimiter": "\\t"})
+        'tags[2    ]: foo    bar'
+
+        >>> encode([1, 2, 3], {"length_marker": "#"})
+        '[#3]: 1,2,3'
+    """
+    raise NotImplementedError("TOON encoder is not yet implemented")
diff --git a/src/toon_format/py.typed b/src/toon_format/py.typed
diff --git a/src/toon_format/types.py b/src/toon_format/types.py
@@ -0,0 +1,37 @@
+"""Type definitions for TOON encoder and decoder."""
+
+from __future__ import annotations
+
+from typing import Any, Literal, TypeAlias, TypedDict
+
+# JSON-compatible types
+JsonPrimitive: TypeAlias = str | int | float | bool | None
+JsonValue: TypeAlias = JsonPrimitive | dict[str, "JsonValue"] | list["JsonValue"]
+JsonObject: TypeAlias = dict[str, JsonValue]
+JsonArray: TypeAlias = list[JsonValue]
+
+
+class EncodeOptions(TypedDict, total=False):
+    """Options for encoding values to TOON format.
+
+    Attributes:
+        indent: Number of spaces per indentation level (default: 2)
+        delimiter: Delimiter for array values and tabular rows (default: ',')
+        length_marker: Optional marker to prefix array lengths (default: False)
+    """
+
+    indent: int
+    delimiter: Literal[",", "\t", "|"]
+    length_marker: Literal["#", False]
+
+
+class DecodeOptions(TypedDict, total=False):
+    """Options for decoding TOON format to values.
+
+    Attributes:
+        indent: Expected number of spaces per indentation level (default: 2)
+        strict: Enable strict validation (default: True)
+    """
+
+    indent: int
+    strict: bool
diff --git a/tests/__init__.py b/tests/__init__.py
@@ -0,0 +1 @@
+"""Tests for the TOON package."""
diff --git a/tests/test_decoder.py b/tests/test_decoder.py
@@ -0,0 +1,67 @@
+"""Tests for the TOON decoder."""
+
+import pytest
+
+from toon_format import decode
+
+
+def test_decode_not_implemented():
+    """Test that decode raises NotImplementedError."""
+    with pytest.raises(NotImplementedError, match="not yet implemented"):
+        decode("key: value")
+
+
+def test_decode_with_options_not_implemented():
+    """Test that decode with options raises NotImplementedError."""
+    with pytest.raises(NotImplementedError, match="not yet implemented"):
+        decode("[3]: 1,2,3", {"strict": False})
+
+
+# Placeholder tests for future implementation
+@pytest.mark.skip(reason="Implementation pending")
+def test_decode_simple_object():
+    """Test decoding a simple object."""
+    toon_data = "id: 123\nname: Ada\nactive: true"
+    result = decode(toon_data)
+    expected = {"id": 123, "name": "Ada", "active": True}
+    assert result == expected
+
+
+@pytest.mark.skip(reason="Implementation pending")
+def test_decode_array_of_objects():
+    """Test decoding a tabular array."""
+    toon_data = "items[2]{sku,qty,price}:\n  A1,2,9.99\n  B2,1,14.5"
+    result = decode(toon_data)
+    expected = {
+        "items": [
+            {"sku": "A1", "qty": 2, "price": 9.99},
+            {"sku": "B2", "qty": 1, "price": 14.5},
+        ]
+    }
+    assert result == expected
+
+
+@pytest.mark.skip(reason="Implementation pending")
+def test_decode_primitive_array():
+    """Test decoding a primitive array."""
+    toon_data = "tags[3]: foo,bar,baz"
+    result = decode(toon_data)
+    expected = {"tags": ["foo", "bar", "baz"]}
+    assert result == expected
+
+
+@pytest.mark.skip(reason="Implementation pending")
+def test_decode_root_array():
+    """Test decoding a root-level array."""
+    toon_data = "[3]: 1,2,3"
+    result = decode(toon_data)
+    expected = [1, 2, 3]
+    assert result == expected
+
+
+@pytest.mark.skip(reason="Implementation pending")
+def test_decode_strict_mode():
+    """Test that strict mode validates input."""
+    invalid_toon = "items[3]{id,name}:\n  1,Alice\n  2,Bob"  # Length mismatch
+    with pytest.raises(ValueError, match="length"):
+        decode(invalid_toon, {"strict": True})
diff --git a/tests/test_encoder.py b/tests/test_encoder.py
