[Quantization] Attention/ KV Cache Refactor (#1651)

## Purpose ##
* Support fully-expressive attention and kv cache quantization
* Support running kv cache quantization evals with hf transformers
* Resolves #1949
* Resolves #1928

```python3
recipe = QuantizationModifier(
    config_groups={
        "attention": QuantizationScheme(
            targets=["LlamaAttention"],
            input_activations=QuantizationArgs(
                num_bits=8, type="float", strategy="tensor"
            ),
        )
    }
)
```

```json
{
  "quantization_config": {
    "config_groups": {
      "group_0": {
        "format": null,
        "input_activations": {
          "dynamic": false,
          "num_bits": 8,
          "observer": "minmax",
          "strategy": "tensor",
          "symmetric": true,
          "type": "float"
        },
        "output_activations": null,
        "targets": [
          "LlamaAttention"
        ],
        "weights": null
      }
    },
    "format": "dense",
    "ignore": [],
    "kv_cache_scheme": {
      "dynamic": false,
      "group_size": null,
      "num_bits": 8,
      "observer": "minmax",
      "strategy": "tensor",
      "symmetric": true,
      "type": "float"
    },
    "quant_method": "compressed-tensors",
    "quantization_status": "frozen",
  },
}
```

## Prerequisites ##
* Must be merged at the same time as
vllm-project/compressed-tensors#436

## Changes ##
* Replace hooks
* Remove `calibrate_kv_cache_input_hook`,
`calibrate_kv_cache_output_hook`, `initialize_quantized_kv_cache`
* Add `calibrate_query_hook` `calibrate_key_hook`,
`calibrate_value_hook`
* QuantizationMixin now initializes "q", "k", and "v" obsevers
([depending on the attached
submodules](https://github.com/vllm-project/llm-compressor/pull/1651/files#diff-33303ae48e185b2fbb14dc45c2052805837deb5723248367b9579321c4c4e974R263-R270))
and adds the appropriate hooks

* Miscellaneous
  * Fix minor shape bug in `_flatten_attention`
  * Add support for "attn_head" strategy in `_flatten_attention`

* Tests
* Removed old QuantizationKVCache tests (these classes are now tested
[here])(https://github.com/neuralmagic/compressed-tensors/pull/436/files#diff-6e33ff48047dc4f7c9d969293f87e32e4d5ec3f3e8b741ea757780c8c0aab775)
  * Updated scale names to avoid using enum
  * Avoid unnecessary tokenization to reduce runtime

## Testing ##
* Kv cache regression tests pass
* Able to quantize attention with scripts (will add to examples once
loadable in vllm)
  * kylesayrs/Llama-3.2-1B-Instruct-attention-fp8-head
  * kylesayrs/Llama-3.2-1B-Instruct-attention-nvfp4-head
* Nightly passes (in progress)

## Evaluation ##
<details><summary>eval.py</summary>

```python
import sys
import lm_eval

model_id = sys.argv[1]

print(model_id)
results = lm_eval.simple_evaluate(
    # 3) hf serialized
    model="hf",
    model_args={
        "pretrained": model_id,
        "add_bos_token": False,
        "dtype": "auto",
        "device_map": "cuda",
        #"max_length": 128000,
    },
    device="cuda",
    # 3/)

    #tasks=["gsm8k_platinum", "mmlu_llama", "longbench2_single"],
    tasks=["gsm8k_platinum"],
    batch_size=64,
    apply_chat_template=True,
    fewshot_as_multiturn=True,
)
print(model_id)
print(lm_eval.utils.make_table(results))
```

</details>
    
<details><summary>compress.py</summary>

```python
from datasets import load_dataset
from transformers import AutoModelForCausalLM, AutoTokenizer

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.utils import dispatch_for_generation
from compressed_tensors.quantization import QuantizationScheme, QuantizationArgs

# Select model and load it.
#model_id = "Qwen/Qwen2.5-14B-Instruct-1M"
model_id = "meta-llama/Llama-3.1-8B-Instruct"
model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype="auto")
tokenizer = AutoTokenizer.from_pretrained(model_id)

# Select calibration dataset.
DATASET_ID = "ultrachat_200k"
DATASET_SPLIT = "train_sft"

# Select number of samples. 512 samples is a good place to start.
# Increasing the number of samples can improve accuracy.
NUM_CALIBRATION_SAMPLES = 512
MAX_SEQUENCE_LENGTH = 2048

# Configure the quantization algorithm to run.
args = QuantizationArgs(
    num_bits=8,
    type="float",
    strategy="attn_head",
    symmetric=True,
    observer="static_minmax",
)
recipe = QuantizationModifier(
    # config_groups={
    #     "attention": QuantizationScheme(
    #         #targets=["Qwen2Attention"],
    #         targets=["LlamaAttention"],
    #         input_activations=args,
    #     )
    # }
    kv_cache_scheme=args,
)

# Apply algorithms.
oneshot(
    model=model,
    dataset=DATASET_ID,
    splits={"calibration": f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]"},
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
)

# Confirm generations of the quantized model look sane.
print("\n\n")
print("========== SAMPLE GENERATION ==============")
dispatch_for_generation(model)
sample = tokenizer("Hello my name is", return_tensors="pt")
sample = {key: value.to(model.device) for key, value in sample.items()}
output = model.generate(**sample, max_new_tokens=100)
print(tokenizer.decode(output[0]))
print("==========================================\n\n")

# Save to disk compressed.
SAVE_DIR = model_id.rstrip("/").split("/")[-1] + f"-KV-FP8-{args.strategy}-{args.observer}"
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)
```

</details>

Model | GSM8K
-- | --
nm-testing/Llama-3.1-8B-Instruct | 0.8337
nm-testing/Llama-3.1-8B-Instruct-KV-FP8-Tensor | 0.8271
nm-testing/Llama-3.1-8B-Instruct-KV-FP8-Head | 0.8354
nm-testing/Llama-3.1-8B-Instruct-QKV-FP8-Tensor | 0.8321
nm-testing/Llama-3.1-8B-Instruct-QKV-FP8-Head | 0.8238

---------

Signed-off-by: Kyle Sayers <[email protected]>

Author: kylesayrs

experimental/llama3_attention.py

@@ -0,0 +1,87 @@
+from datasets import load_dataset
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from llmcompressor import oneshot
+from llmcompressor.modifiers.quantization import QuantizationModifier
+from llmcompressor.utils import dispatch_for_generation
+from compressed_tensors.quantization import QuantizationScheme, QuantizationArgs
+
+# Select model and load it.
+model_id = "meta-llama/Meta-Llama-3-8B-Instruct"
+model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype="auto")
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+# Select calibration dataset.
+DATASET_ID = "HuggingFaceH4/ultrachat_200k"
+DATASET_SPLIT = "train_sft"
+
+# Select number of samples. 512 samples is a good place to start.
+# Increasing the number of samples can improve accuracy.
+NUM_CALIBRATION_SAMPLES = 512
+MAX_SEQUENCE_LENGTH = 2048
+
+# Load dataset and preprocess.
+ds = load_dataset(DATASET_ID, split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES}]")
+ds = ds.shuffle(seed=42)
+
+
+def preprocess(example):
+    return {
+        "text": tokenizer.apply_chat_template(
+            example["messages"],
+            tokenize=False,
+        )
+    }
+
+
+ds = ds.map(preprocess)
+
+
+# Tokenize inputs.
+def tokenize(sample):
+    return tokenizer(
+        sample["text"],
+        padding=False,
+        max_length=MAX_SEQUENCE_LENGTH,
+        truncation=True,
+        add_special_tokens=False,
+    )
+
+
+ds = ds.map(tokenize, remove_columns=ds.column_names)
+
+# Configure the quantization algorithm to run.
+recipe = QuantizationModifier(
+    config_groups={
+        "attention": QuantizationScheme(
+            targets=["LlamaAttention"],
+            input_activations=QuantizationArgs(
+                num_bits=8, type="float", strategy="attn_head"
+            ),
+        )
+    }
+)
+
+# Apply algorithms.
+oneshot(
+    model=model,
+    dataset=ds,
+    recipe=recipe,
+    max_seq_length=MAX_SEQUENCE_LENGTH,
+    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
+)
+
+# Confirm generations of the quantized model look sane.
+print("\n\n")
+print("========== SAMPLE GENERATION ==============")
+dispatch_for_generation(model)
+sample = tokenizer("Hello my name is", return_tensors="pt")
+sample = {key: value.to(model.device) for key, value in sample.items()}
+output = model.generate(**sample, max_new_tokens=100)
+print(tokenizer.decode(output[0]))
+print("==========================================\n\n")
+
+# Save to disk compressed.
+SAVE_DIR = model_id.rstrip("/").split("/")[-1] + "-attention-fp8-head"
+model.save_pretrained(SAVE_DIR, save_compressed=True)
+tokenizer.save_pretrained(SAVE_DIR)

src/llmcompressor/modifiers/quantization/__init__.py

@@ -1,5 +1,4 @@
 # ruff: noqa
 
-from .cache import *
 from .gptq import *
 from .quantization import *