Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dev: provide ability to set timeout options #109

Merged
merged 3 commits into from
Feb 22, 2025
Merged

Dev: provide ability to set timeout options #109

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
96a87a6
support configuration of formulation timeout
Chenglong-MS Feb 21, 2025
77ae246
bump version
Chenglong-MS Feb 21, 2025
81a3cc0
thanks copilot reviewer, I'm addressing my logger issues...
Chenglong-MS Feb 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions py-src/data_formulator/agents/agent_data_rec.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,14 @@ def process_gpt_response(self, input_tables, messages, response):
result['refined_goal'] = refined_goal
candidates.append(result)

logger.info("=== Recommendation Candidates ===>")
for candidate in candidates:
for key, value in candidate.items():
if key in ['dialog', 'content']:
logger.info(f"##{key}:\n{str(value)[:1000]}...")
else:
logger.info(f"## {key}:\n{value}")

return candidates


Expand Down
16 changes: 7 additions & 9 deletions py-src/data_formulator/agents/agent_data_transform_v2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,16 +224,9 @@ def process_gpt_response(self, input_tables, messages, response):

code_blocks = extract_code_from_gpt_response(choice.message.content + "\n", "python")

logger.info("=== Code blocks ===>")
logger.info(code_blocks)

if len(code_blocks) > 0:
code_str = code_blocks[-1]

for table in input_tables:
logger.info(f"Table: {table['name']}")
logger.info(table['rows'])

try:
result = py_sandbox.run_transform_in_sandbox2020(code_str, [t['rows'] for t in input_tables])
result['code'] = code_str
Expand All @@ -256,8 +249,13 @@ def process_gpt_response(self, input_tables, messages, response):
result['refined_goal'] = refined_goal
candidates.append(result)

logger.info("=== Candidates ===>")
logger.info(candidates)
logger.info("=== Transform Candidates ===>")
for candidate in candidates:
for key, value in candidate.items():
if key in ['dialog', 'content']:
logger.info(f"##{key}:\n{str(value)[:1000]}...")
else:
logger.info(f"## {key}:\n{value}")

return candidates

Expand Down
42 changes: 28 additions & 14 deletions py-src/data_formulator/app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,8 +213,8 @@ def test_model():
content = request.get_json()

# contains endpoint, key, model, api_base, api_version
print("content------------------------------")
print(content)
logger.info("content------------------------------")
logger.info(content)

client = get_client(content['model'])

Expand All @@ -226,8 +226,8 @@ def test_model():
]
)

print(f"model: {content['model']}")
print(f"welcome message: {response.choices[0].message.content}")
logger.info(f"model: {content['model']}")
logger.info(f"welcome message: {response.choices[0].message.content}")

if "I can hear you." in response.choices[0].message.content:
result = {
Expand All @@ -236,7 +236,7 @@ def test_model():
"message": ""
}
except Exception as e:
print(f"Error: {e}")
logger.info(f"Error: {e}")
error_message = str(e)
result = {
"model": content['model'],
Expand All @@ -250,13 +250,13 @@ def test_model():

@app.route("/", defaults={"path": ""})
def index_alt(path):
print(app.static_folder)
logger.info(app.static_folder)
return send_from_directory(app.static_folder, "index.html")

@app.errorhandler(404)
def page_not_found(e):
# your processing here
print(app.static_folder)
logger.info(app.static_folder)
return send_from_directory(app.static_folder, "index.html") #'Hello 404!' #send_from_directory(app.static_folder, "index.html")

###### test functions ######
Expand Down Expand Up @@ -425,14 +425,21 @@ def derive_data():
new_fields = content["new_fields"]
instruction = content["extra_prompt"]

max_repair_attempts = content["max_repair_attempts"] if "max_repair_attempts" in content else 1

if "additional_messages" in content:
prev_messages = content["additional_messages"]
else:
prev_messages = []

print("spec------------------------------")
print(new_fields)
print(instruction)
logger.info("== input tables ===>")
for table in input_tables:
logger.info(f"===> Table: {table['name']} (first 5 rows)")
logger.info(table['rows'][:5])

logger.info("== user spec ===")
logger.info(new_fields)
logger.info(instruction)

mode = "transform"
if len(new_fields) == 0:
Expand All @@ -447,7 +454,7 @@ def derive_data():
results = agent.run(input_tables, instruction, [field['name'] for field in new_fields], prev_messages)

repair_attempts = 0
while results[0]['status'] == 'error' and repair_attempts == 0: # only try once
while results[0]['status'] == 'error' and repair_attempts < max_repair_attempts: # try up to n times
error_message = results[0]['content']
new_instruction = f"We run into the following problem executing the code, please fix it:\n\n{error_message}\n\nPlease think step by step, reflect why the error happens and fix the code so that no more errors would occur."

Expand Down Expand Up @@ -482,16 +489,23 @@ def refine_data():
output_fields = content["output_fields"]
dialog = content["dialog"]
new_instruction = content["new_instruction"]
max_repair_attempts = content["max_repair_attempts"] if "max_repair_attempts" in content else 1

logger.info("== input tables ===>")
for table in input_tables:
logger.info(f"===> Table: {table['name']} (first 5 rows)")
logger.info(table['rows'][:5])

print("previous dialog")
print(dialog)
logger.info("== user spec ===>")
logger.info(output_fields)
logger.info(new_instruction)

# always resort to the data transform agent
agent = DataTransformationAgentV2(client=client)
results = agent.followup(input_tables, dialog, [field['name'] for field in output_fields], new_instruction)

repair_attempts = 0
while results[0]['status'] == 'error' and repair_attempts == 0: # only try once
while results[0]['status'] == 'error' and repair_attempts < max_repair_attempts: # only try once
error_message = results[0]['content']
new_instruction = f"We run into the following problem executing the code, please fix it:\n\n{error_message}\n\nPlease think step by step, reflect why the error happens and fix the code so that no more errors would occur."
prev_dialog = results[0]['dialog']
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "data_formulator"
version = "0.1.6"
version = "0.1.6.1"

requires-python = ">=3.9"
authors = [
Expand Down
123 changes: 119 additions & 4 deletions src/app/App.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import {
ToggleButton,
Menu,
MenuItem,
TextField,
} from '@mui/material';


Expand All @@ -46,7 +47,7 @@ import { DataFormulatorFC } from '../views/DataFormulator';

import GridViewIcon from '@mui/icons-material/GridView';
import ViewSidebarIcon from '@mui/icons-material/ViewSidebar';

import SettingsIcon from '@mui/icons-material/Settings';
import {
createBrowserRouter,
RouterProvider,
Expand Down Expand Up @@ -295,10 +296,122 @@ const ResetDialog: React.FC = () => {
);
};

const ConfigDialog: React.FC = () => {
const [open, setOpen] = useState(false);
const dispatch = useDispatch();
const config = useSelector((state: DataFormulatorState) => state.config);

const [formulateTimeoutSeconds, setFormulateTimeoutSeconds] = useState(config.formulateTimeoutSeconds);
const [maxRepairAttempts, setMaxRepairAttempts] = useState(config.maxRepairAttempts);

// Add check for changes
const hasChanges = formulateTimeoutSeconds !== config.formulateTimeoutSeconds ||
maxRepairAttempts !== config.maxRepairAttempts;

return (
<>
<Button variant="text" sx={{textTransform: 'none'}} onClick={() => setOpen(true)} startIcon={<SettingsIcon />}>
<Box component="span" sx={{lineHeight: 1.2, display: 'flex', flexDirection: 'column', alignItems: 'left'}}>
<Box component="span" sx={{py: 0, my: 0, fontSize: '10px', mr: 'auto'}}>timeout={config.formulateTimeoutSeconds}s</Box>
<Box component="span" sx={{py: 0, my: 0, fontSize: '10px', mr: 'auto'}}>max_repair={config.maxRepairAttempts}</Box>
</Box>
</Button>
<Dialog onClose={() => setOpen(false)} open={open}>
<DialogTitle>Data Formulator Configuration</DialogTitle>
<DialogContent>
<DialogContentText>
<Box sx={{
display: 'flex',
flexDirection: 'column',
gap: 3,
my: 2,
maxWidth: 400
}}>
<Box sx={{ display: 'flex', alignItems: 'center', gap: 2 }}>
<Box sx={{ flex: 1 }}>
<TextField
label="formulate timeout (seconds)"
type="number"
variant="outlined"
value={formulateTimeoutSeconds}
onChange={(e) => {
const value = parseInt(e.target.value);
setFormulateTimeoutSeconds(value);
}}
inputProps={{
min: 0,
max: 3600,
}}
error={formulateTimeoutSeconds <= 0 || formulateTimeoutSeconds > 3600}
helperText={formulateTimeoutSeconds <= 0 || formulateTimeoutSeconds > 3600 ?
"Value must be between 1 and 3600 seconds" : ""}
fullWidth
/>
<Typography variant="caption" color="text.secondary" sx={{ mt: 1, display: 'block' }}>
Maximum time allowed for the formulation process before timing out.
</Typography>
<Typography variant="caption" color="text.secondary" sx={{ mt: 1, display: 'block' }}>
Smaller values (&lt;30s) make the model fails fast thus providing a smoother UI experience. Increase this value for slow models.
</Typography>
</Box>
</Box>
<Box sx={{ display: 'flex', alignItems: 'center', gap: 2 }}>
<Box sx={{ flex: 1 }}>
<TextField
label="max repair attempts"
type="number"
variant="outlined"
value={maxRepairAttempts}
onChange={(e) => {
const value = parseInt(e.target.value);
setMaxRepairAttempts(value);
}}
fullWidth
inputProps={{
min: 1,
max: 5,
}}
error={maxRepairAttempts <= 0 || maxRepairAttempts > 5}
helperText={maxRepairAttempts <= 0 || maxRepairAttempts > 5 ?
"Value must be between 1 and 5" : ""}
/>
<Typography variant="caption" color="text.secondary" sx={{ mt: 1, display: 'block' }}>
Maximum number of times the LLM will attempt to repair code if generated code fails to execute (recommended = 1).
</Typography>
<Typography variant="caption" color="text.secondary" sx={{ mt: 1, display: 'block' }}>
Higher values might slightly increase the chance of success but can crash the backend. Repair time is as part of the formulate timeout.
</Typography>
</Box>
</Box>
</Box>
</DialogContentText>
</DialogContent>
<DialogActions sx={{'.MuiButton-root': {textTransform: 'none'}}}>
<Button sx={{marginRight: 'auto'}} onClick={() => {
setFormulateTimeoutSeconds(30);
setMaxRepairAttempts(1);
}}>Reset to default</Button>
<Button onClick={() => setOpen(false)}>Cancel</Button>
<Button
variant={hasChanges ? "contained" : "text"}
disabled={!hasChanges || isNaN(maxRepairAttempts) || maxRepairAttempts <= 0 || maxRepairAttempts > 5 || isNaN(formulateTimeoutSeconds) || formulateTimeoutSeconds <= 0 || formulateTimeoutSeconds > 3600}
onClick={() => {
dispatch(dfActions.setConfig({formulateTimeoutSeconds, maxRepairAttempts}));
setOpen(false);
}}
>
Apply
</Button>
</DialogActions>
</Dialog>
</>
);
}

export const AppFC: FC<AppFCProps> = function AppFC(appProps) {

const visViewMode = useSelector((state: DataFormulatorState) => state.visViewMode);
const betaMode = useSelector((state: DataFormulatorState) => state.betaMode);
const config = useSelector((state: DataFormulatorState) => state.config);
const tables = useSelector((state: DataFormulatorState) => state.tables);

// if the user has logged in
Expand Down Expand Up @@ -410,7 +523,7 @@ export const AppFC: FC<AppFCProps> = function AppFC(appProps) {

let appBar = [
<AppBar className="app-bar" position="static" key="app-bar-main">
<Toolbar variant="dense" sx={{ backgroundColor: betaMode ? 'lavender' : '' }}>
<Toolbar variant="dense">
<Button href={"/"} sx={{
display: "flex", flexDirection: "row", textTransform: "none",
backgroundColor: 'transparent',
Expand All @@ -420,7 +533,7 @@ export const AppFC: FC<AppFCProps> = function AppFC(appProps) {
}} color="inherit">
<Box component="img" sx={{ height: 32, marginRight: "12px" }} alt="" src={dfLogo} />
<Typography variant="h6" noWrap component="h1" sx={{ fontWeight: 300, display: { xs: 'none', sm: 'block' } }}>
{toolName} {betaMode ? "β" : ""} {process.env.NODE_ENV == "development" ? "" : ""}
{toolName} {process.env.NODE_ENV == "development" ? "" : ""}
</Typography>
</Button>
<Box sx={{ flexGrow: 1, textAlign: 'center', display: 'flex', justifyContent: 'center' }} >
Expand All @@ -432,6 +545,8 @@ export const AppFC: FC<AppFCProps> = function AppFC(appProps) {
about
</Button>
<Divider orientation="vertical" variant="middle" flexItem /> */}
<ConfigDialog />
<Divider orientation="vertical" variant="middle" flexItem />
<ModelSelectionButton />
<Divider orientation="vertical" variant="middle" flexItem />
<Typography sx={{ display: 'flex', fontSize: 14, alignItems: 'center', gap: 1 }}>
Expand Down
22 changes: 16 additions & 6 deletions src/app/dfSlice.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,10 @@ export interface DataFormulatorState {

chartSynthesisInProgress: string[];

betaMode: boolean;
config: {
formulateTimeoutSeconds: number;
maxRepairAttempts: number;
}
}

// Define the initial state using that type
Expand Down Expand Up @@ -98,7 +101,10 @@ const initialState: DataFormulatorState = {

chartSynthesisInProgress: [],

betaMode: false,
config: {
formulateTimeoutSeconds: 30,
maxRepairAttempts: 1,
}
}

let getUnrefedDerivedTableIds = (state: DataFormulatorState) => {
Expand Down Expand Up @@ -247,7 +253,11 @@ export const dataFormulatorSlice = createSlice({

state.chartSynthesisInProgress = [];

state.betaMode = false;
// avoid resetting config
// state.config = {
// formulateTimeoutSeconds: 30,
// repairAttempts: 1,
// }
},
loadState: (state, action: PayloadAction<any>) => {

Expand All @@ -274,10 +284,10 @@ export const dataFormulatorSlice = createSlice({

state.chartSynthesisInProgress = [];

state.betaMode = savedState.betaMode;
state.config = savedState.config;
},
toggleBetaMode: (state, action: PayloadAction<boolean>) => {
state.betaMode = action.payload;
setConfig: (state, action: PayloadAction<{formulateTimeoutSeconds: number, maxRepairAttempts: number}>) => {
state.config = action.payload;
},
selectModel: (state, action: PayloadAction<string | undefined>) => {
state.selectedModelId = action.payload;
Expand Down
Loading