generation_template.py

from dataclasses import dataclass

# Base config
@dataclass
class BaseTemplate:
    S_TOKEN = "USER"
    E_TOKEN = "ASSISTANT"
    SEP = '\n'
    BOI_TOKEN = '<img>'
    EOI_TOKEN = '</img>'
    IMG_TOKEN = '<img_{:05d}>'
    NUM_IMG_TOKNES = 32
    NUM_IMG_CODES = 8192
    image_id_shift = 32000
    vocab = {
        'boi_token': None,
        'eoi_token': None,   
        'image_token_length': 32,

        'text_token_s': 1,           # bos token
        'text_token_e': 2,           # eos token
        'text_token_p': 0,           # pad token

        'image_vocab_size': 8192,
        'text_vocab_size': 32000    # 32000 (original LLaMA)      
    }
  

# Element
@dataclass
class ElementTemplate:
    gen_cfg = {
        'temperature': 0.7,
        'num_beams': 1,
        'max_new_tokens': 256,
        'top_p':  0.8,
        'do_sample': True
    }
    object_prompt = 'You are an assistant dedicated to generating common object spanning various categories, including animals, plants, fruits, household items, clothing, vehicles, food, musical instruments, and electronic devices consisting of only one word.\nUSER: Generate ten common object. output format must be 1.object, 2. object, 3. object...\nASSISTANT:'
    color_prompt = 'You are an assistant dedicated to generating common color spanning various categories, including animals, plants, fruits, household items, clothing, vehicles, food, musical instruments, and electronic devices consisting of only one word.\nUSER: Generate twenty common color consisting of only one word.\nASSISTANT:'
    texture_prompt = 'You are an assistant dedicated to generating diverse textures spanning various categories, including animals, plants, fruits, household items, clothing, vehicles, food, musical instruments, and electronic devices.\nUSER: Generate thirty diverse textures consisting of only one adjective.\nASSISTANT:'
    shape_prompt = 'You are an assistant dedicated to generating common shape spanning various categories, including animals, plants, fruits, household items, clothing, vehicles, food, musical instruments, and electronic devices.\nUSER: Generate a list of common shape consisting of only one adjective.\nASSISTANT:'
    spatial_prompt = "You are an assistant dedicated to generating spatial relations for describing object positions in 2D and 3D space. Please provide concise relation words or short phrases, without any explanations, context, or extra details.\nUSER: Generate a list of 20 spatial relations.\nASSISTANT:"

# Prompt
@dataclass
class PromptTemplate:
    gen_cfg = {
        'temperature': 1.0,
        'num_beams': 1,
        'max_new_tokens': 256,
        'top_p':  0.9,
        'do_sample': True
    }
    non_spatial_prompt = 'You are an assistant dedicated to generating natural prompts that contain subjects and objects by using nonspatial relationship words such as wear, watch, speak, hold, have, run, look at, talk to, jump, play, walk with, stand on, and sit on.\n\nUSER: Generate a prompt that contains subjects and objects by using non-spatial relationship words.\nASSISTANT: Two friends are watching a movie together on a large TV screen.\n\nUSER: Generate a prompt that contains subjects and objects by using non-spatial relationship words.\nASSISTANT: Two athletes are running along the beach as the sun sets behind them.\n\nUSER: Generate a prompt that contains subjects and objects by using non-spatial relationship words.\nASSISTANT:'
    complex_prompt = 'You are an assistant dedicated to generating natural compositional phrases or prompts, containing multiple objects (number ≥ 2) with one or several adjectives from color, shape, and texture descriptions and spatial (left/right/top/bottom/next to/near/on side of) or nonspatial relationships.\n\nUSER: Please generate a compositional phrase or sentence containing multiple objects with one or several adjectives and relationships.\nASSISTANT: The fluffy white cat sat next to the black leather couch.\n\nUSER: Please generate a compositional phrase or sentence containing multiple objects with one or several adjectives and relationships.\nASSISTANT: The sleek black phone rested beside the textured brown leather wallet.\n\nUSER: Please generate a compositional phrase or sentence containing multiple objects with one or several adjectives and relationships.\nASSISTANT: The red spherical balloon floated above the striped rectangular kite and the green triangular flag.\n\nUSER: Please generate a compositional phrase or sentence containing multiple objects with one or several adjectives and relationships.\nASSISTANT: The golden, sunlit leaves floated softly above the jagged, rust-colored rocks, their delicate, lacy shapes casting playful shadows on the uneven ground.\n\nUSER: Please generate a compositional phrase or sentence containing multiple objects with one or several adjectives and relationships.\nASSISTANT:'

# Image
@dataclass
class ImageTemplate:
    '''
    gen_cfg = {
        'temperature': 0.8,
        'num_beams': 5,
        'max_new_tokens': 256,
        'top_p':  0.8,
        'do_sample': True
    }
    '''
    gen_cfg = {
        'temperature': 0.4,
        'max_new_tokens': 256,
        'top_p':  0.5,
        'do_sample': True
    }
    # .format(prompt)
    image_prompt = 'USER: {} Please generate an image.\nASSISTANT:'

# VQA
@dataclass
class VQATemplate:
    gen_config = {
        'temperature': 1.0,
        'num_beams': 5,
        'max_new_tokens': 256,
        'top_p':  0.9,
        'do_sample': True
    }  
    
    # .format(prompt)
    attribute_prompt = 'You are an assistant dedicated to transforming a sentence into several questions. You should first divide it into simple concepts and relations, and then provide the corresponding questions. Avoid using pronouns, such as he, she, it, and they.\n\nUSER: A white harp and a rust soup.\nASSISTANT: Concepts and relations: a white harp, a rust soup; Questions: Is there a white harp? Is there a rust soup?\n\nUSER: A quarter circle lily and a hexagon mirror.\nASSISTANT: Concepts and relations: a quarter circle lily, a hexagon mirror; Questions: Is there a quarter-circle lily? Is there a hexagon mirror?\n\nUSER: Shiny mop and metal key holder.\nASSISTANT: Concepts and relations: a shiny mop, a metal key holder; Questions: Is there a shiny mop? Is there a metal key holder?\n\nUSER: {}\nASSISTANT:'
    layout_prompt = 'You are an assistant dedicated to transforming a sentence into several questions. You should first divide it into simple concepts and relations, and then provide the corresponding questions. Avoid using pronouns, such as he, she, it, and they.\n\nUSER: A pancake on the left of a pasta.\nASSISTANT: Concepts and relations: a pancake, a pasta, a pancake is on the left of a pasta; Questions: Is there a pancake? Is there a pasta? Is a pancake on the left of a pasta?\n\nUSER: A lamp behind a screwdriver.\nASSISTANT: Concepts and relations: a lamp, a screwdriver, a lamp is behind a screwdriver; Questions: Is there a lamp? Is there a screwdriver? Is a lamp behind a screwdriver?\n\nUSER: Three light bulbs and eight pumpkins.\nASSISTANT: Concepts and relations: three light bulbs, eight pumpkins; Questions: Are there three light bulbs? Are there eight pumpkins?\n\nUSER: {}\nASSISTANT:'
    non_spatial_complex_prompt = 'You are an assistant dedicated to transforming a sentence into several questions. You should first divide it into simple concepts and relations, and then provide the corresponding questions. Avoid using pronouns, such as he, she, it, and they.\n\nUSER: A chef is holding a knife and preparing a dish on the stove.\nASSISTANT: Concepts and relations: a chef, a knife, a dish, the stove, a chef is holding a knife, a chef is preparing a dish; Questions: Is there a chef? Is there a knife? Is there a dish? Is there a stove? Is a chef holding a knife? Is a chef preparing a dish?\n\nUSER: The green teapot is located near the round oak table.\nASSISTANT: Concepts and relations: a green teapot, a round oak table, the green teapot is near the round oak table, the round oak table is near the green teapot; Questions: Is there a green teapot? Is there a round oak table? Is the green teapot near the round oak table? Is the round oak table near the green teapot?\n\nUSER: The chunky wooden lamp casts a warm glow on the tattered blue curtains.\nASSISTANT: Concepts and relations: a chunky wooden lamp, a warm glow, tattered blue curtains, a chunky wooden lamp casts a warm glow, the warm glow is on the tattered blue curtains; Questions: Is there a chunky wooden lamp? Is there a warm glow? Are there tattered blue curtains? Is a chunky wooden lamp casting a warm glow? Is the warm glow on the tattered blue curtains?\n\nUSER: The vibrant orange tomato sat atop the crisp green leaf and the juicy red watermelon.\nASSISTANT: Concepts and relations: a vibrant orange tomato, a crisp green leaf, a juicy red watermelon, a vibrant orange tomato is atop a crisp green leaf, a vibrant orange tomato is atop a juicy red watermelon; Questions: Is there a vibrant orange tomato? Is there a crisp green leaf? Is there a juicy red watermelon? Is the vibrant orange tomato atop the crisp green leaf? Is the vibrant orange tomato atop the juicy red watermelon?\n\nUSER: {}\nASSISTANT:'