From b4016ef9bd9c8cd2012dcda02f15a8a4be6b0199 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Mon, 23 Sep 2024 17:49:49 -0700 Subject: [PATCH 01/39] updates to enable smarter data load --- app.py | 51 ++---- server/agents/agent_data_load.py | 91 +++++++--- server/agents/agent_sort_data.py | 2 +- server/agents/agent_utils.py | 8 +- src/app/App.tsx | 2 +- src/app/dfSlice.tsx | 13 +- src/app/utils.tsx | 2 +- src/components/ChartTemplates.tsx | 62 ++++++- src/components/ComponentType.tsx | 2 +- src/views/EncodingBox.tsx | 274 +++--------------------------- 10 files changed, 179 insertions(+), 328 deletions(-) diff --git a/app.py b/app.py index 35eee8b..57b4f1c 100644 --- a/app.py +++ b/app.py @@ -61,14 +61,6 @@ def get_example_dataset_list(): except: pass - # this is a dataset we use for demoing the system - try: - with open('global-energy.json', 'r') as f: - info_obj = {'name': 'global-energy.csv', 'snapshot': json.dumps(json.load(f))} - dataset_info.append(info_obj) - except: - pass - response = flask.jsonify(dataset_info) response.headers.add('Access-Control-Allow-Origin', '*') return response @@ -76,14 +68,9 @@ def get_example_dataset_list(): @app.route('/vega-dataset/') def get_datasets(path): try: - # this is a dataset we use for demoing the system - if path == "global-energy.csv": - with open('global-energy.json', 'r') as f: - data_object = json.dumps(json.load(f)) - else: - df = vega_data(path) - # to_json is necessary for handle NaN issues - data_object = df.to_json(None, 'records') + df = vega_data(path) + # to_json is necessary for handle NaN issues + data_object = df.to_json(None, 'records') except Exception as err: print(path) print(err) @@ -407,29 +394,21 @@ def refine_data(): print("previous dialog") print(dialog[0]['content']) - prev_system_prompt = dialog[0]['content'] - - if prev_system_prompt.startswith("You are a data scientist to help user to filter data based on user description."): - agent = DataFilterAgent(client, model=model) - results = agent.followup(input_tables[0], dialog, new_instruction) - elif prev_system_prompt.startswith("You are a data scientist to help user to derive new column based on existing columns in a dataset."): - agent = GenericPyConceptDeriveAgent(client, model=model) - new_field_name = [field['name'] for field in output_fields if field['name'] not in input_tables[0][0].keys()][0] - results = agent.followup(input_tables[0], new_field_name, dialog, new_instruction) - else: - agent = DataTransformationAgentV2(client, model=model) - results = agent.followup(input_tables, dialog, [field['name'] for field in output_fields], new_instruction) - repair_attempts = 0 - while results[0]['status'] == 'error' and repair_attempts < 2: - error_message = results[0]['content'] - new_instruction = f"We run into the following problem executing the code, please fix it:\n\n{error_message}\n\nPlease think step by step, reflect why the error happens and fix the code so that no more errors would occur." + # always resort to the data transform agent + agent = DataTransformationAgentV2(client, model=model) + results = agent.followup(input_tables, dialog, [field['name'] for field in output_fields], new_instruction) - response_message = dialog['response']['choices'][0]['message'] - prev_dialog = [*dialog['messages'], {"role": response_message['role'], 'content': response_message['content']}] + repair_attempts = 0 + while results[0]['status'] == 'error' and repair_attempts < 2: + error_message = results[0]['content'] + new_instruction = f"We run into the following problem executing the code, please fix it:\n\n{error_message}\n\nPlease think step by step, reflect why the error happens and fix the code so that no more errors would occur." - results = agent.followup(input_tables, prev_dialog, [field['name'] for field in output_fields], new_instruction) - repair_attempts += 1 + response_message = dialog['response']['choices'][0]['message'] + prev_dialog = [*dialog['messages'], {"role": response_message['role'], 'content': response_message['content']}] + + results = agent.followup(input_tables, prev_dialog, [field['name'] for field in output_fields], new_instruction) + repair_attempts += 1 response = flask.jsonify({ "status": "ok", "token": token, "results": results}) else: diff --git a/server/agents/agent_data_load.py b/server/agents/agent_data_load.py index b234bd4..6470b05 100644 --- a/server/agents/agent_data_load.py +++ b/server/agents/agent_data_load.py @@ -13,38 +13,67 @@ Given a dataset provided by the user, identify their type and semantic type, and provide a very short summary of the dataset. Types to consider include: string, number, date -Semantic types to consider include: Location, Year, Month, Day, Date, Time, DateTime, Duration, Name, Percentage, String, Number +Semantic types to consider include: Location, Year, Month, Day, Date, Time, DateTime, Range, Duration, Name, Percentage, String, Number + +Furthermore, if the field is string type and is ordinal (especially for english month name, week name, range), provide the natural sort order of the fields here. +Otherwise, put sort_order as null (for example, Name should not be sorted). Create a json object function based off the [DATA] provided. +output should be in the format of: + +```json +{ + "fields": { + "field1": {"type": ..., "semantic_type": ..., "sort_order": [...]}, // replace field1 field2 with actual field names, if the field is string type and is ordinal, provide the natural sort order of the fields here + "field2": {"type": ..., "semantic_type": ..., "sort_order": null}, + ... + }, + "data summary": ... // a short summary of the data +} +``` +''' + +EXAMPLES = ''' [DATA] Here are our datasets, here are their field summaries and samples: -table_0 (us_covid_cases) fields: - Date -- type: object, values: 1/1/2021, 1/1/2022, 1/1/2023, ..., 9/8/2022, 9/9/2020, 9/9/2021, 9/9/2022 - Cases -- type: int64, values: -23999, -14195, -6940, ..., 1018935, 1032159, 1178403, 1433977 +table_0 (income_json) fields: + name -- type: object, values: Alabama, Alaska, Arizona, Arkansas, California, Colorado, Connecticut, Delaware, District of Columbia, Florida, ..., South Dakota, Tennessee, Texas, Utah, Vermont, Virginia, Washington, West Virginia, Wisconsin, Wyoming + region -- type: object, values: midwest, northeast, other, south, west + state_id -- type: int64, values: 1, 2, 4, 5, 6, 8, 9, 10, 11, 12, ..., 47, 48, 49, 50, 51, 53, 54, 55, 56, 72 + pct -- type: float64, values: 0.006, 0.008, 0.02, 0.021, 0.022, 0.024, 0.025, 0.026000000000000002, 0.027, 0.028, ..., 0.192, 0.193, 0.194, 0.196, 0.197, 0.199, 0.2, 0.201, 0.213, 0.289 + total -- type: int64, values: 222679, 250875, 256563, 268015, 291468, 326086, 337245, 405504, 410347, 449296, ..., 3522934, 3721358, 3815532, 4551497, 4763457, 4945140, 7168502, 7214163, 8965352, 12581722 + group -- type: object, values: 10000 to 14999, 100000 to 149999, 15000 to 24999, 150000 to 199999, 200000+, 25000 to 34999, 35000 to 49999, 50000 to 74999, 75000 to 99999, <10000 + +table_0 (income_json) sample: -table_0 (us_covid_cases) sample: ``` -|Date|Cases -0|1/21/2020|1 -1|1/22/2020|0 -2|1/23/2020|0 -3|1/24/2020|1 -4|1/25/2020|1 +|name|region|state_id|pct|total|group +0|Alabama|south|1|0.10200000000000001|1837292|<10000 +1|Alabama|south|1|0.07200000000000001|1837292|10000 to 14999 +2|Alabama|south|1|0.13|1837292|15000 to 24999 +3|Alabama|south|1|0.115|1837292|25000 to 34999 +4|Alabama|south|1|0.14300000000000002|1837292|35000 to 49999 ...... ``` [OUTPUT] +```json { "fields": { - "Date": {"type": "date", "semantic_type": "Date"}, - "Cases": {"type": "number", "semantic_type": "Number"} + "name": {"type": "string", "semantic_type": "Location", "sort_order": null}, + "region": {"type": "string", "semantic_type": "String", "sort_order": ["northeast", "midwest", "south", "west", "other"]}, + "state_id": {"type": "number", "semantic_type": "Number", "sort_order": null}, + "pct": {"type": "number", "semantic_type": "Percentage", "sort_order": null}, + "total": {"type": "number", "semantic_type": "Number", "sort_order": null}, + "group": {"type": "string", "semantic_type": "Range", "sort_order": ["<10000", "10000 to 14999", "15000 to 24999", "25000 to 34999", "35000 to 49999", "50000 to 74999", "75000 to 99999", "100000 to 149999", "150000 to 199999", "200000+"]} }, - "data summary": "US covid 19 data from 2020 to 2022" + "data summary": "The dataset contains information about income distribution across different states in the USA. It includes fields for state names, regions, state IDs, percentage of total income, total income, and income groups." } +``` [DATA] @@ -68,15 +97,27 @@ [OUTPUT] -{ - "fields": { - "Date": {"type": "date", "semantic_type": "Date"}, - "City": {"type": "string", "semantic_type": "Location"}, - "Temperature": {"type": "number", "semantic_type": "Number"} - }, - "data summary": "Seattle and Atlanta temperature in 2020" -} -''' +``` +{ + "fields": { + "Date": { + "type": "string", + "semantic_type": "Date", + "sort_order": null + }, + "City": { + "type": "string", + "semantic_type": "Location", + "sort_order": null + }, + "Temperature": { + "type": "number", + "semantic_type": "Number", + "sort_order": null + } + }, + "data_summary": "This dataset contains weather information for the cities of Seattle and Atlanta. The fields include the date, city name, and temperature readings. The 'Date' field represents dates in a string format, the 'City' field represents city names, and the 'Temperature' field represents temperature values in integer format." +}```''' class DataLoadAgent(object): @@ -86,7 +127,7 @@ def __init__(self, client, model): def run(self, input_data, n=1): - data_summary = generate_data_summary([input_data], include_data_samples=True) + data_summary = generate_data_summary([input_data], include_data_samples=True, field_sample_size=30) user_query = f"[DATA]\n\n{data_summary}\n\n[OUTPUT]" @@ -97,7 +138,7 @@ def run(self, input_data, n=1): ###### the part that calls open_ai response = self.client.chat.completions.create( - model=self.model, messages=messages, temperature=0.2, max_tokens=2400, + model=self.model, messages=messages, temperature=0.2, max_tokens=4096, top_p=0.95, n=n, frequency_penalty=0, presence_penalty=0, stop=None) #log = {'messages': messages, 'response': response.model_dump(mode='json')} diff --git a/server/agents/agent_sort_data.py b/server/agents/agent_sort_data.py index 17b053e..762f607 100644 --- a/server/agents/agent_sort_data.py +++ b/server/agents/agent_sort_data.py @@ -96,7 +96,7 @@ def run(self, name, values, n=1): logger.info("\n=== Sort data agent ===>\n") logger.info(choice.message.content + "\n") - json_blocks = extract_json_objects(choice.message.content + "\n", "json") + json_blocks = extract_json_objects(choice.message.content + "\n") if len(json_blocks) > 0: result = {'status': 'ok', 'content': json_blocks[0]} diff --git a/server/agents/agent_utils.py b/server/agents/agent_utils.py index 829ff72..802112f 100644 --- a/server/agents/agent_utils.py +++ b/server/agents/agent_utils.py @@ -180,7 +180,7 @@ def dedup_data_transform_candidates(candidates): return [items[0] for _, items in candidate_groups.items()] -def get_field_summary(field_name, df): +def get_field_summary(field_name, df, field_sample_size): try: values = sorted([x for x in list(set(df[field_name].values)) if x != None]) except: @@ -188,7 +188,7 @@ def get_field_summary(field_name, df): val_sample = "" - sample_size = 7 + sample_size = field_sample_size if len(values) <= sample_size: val_sample = values @@ -199,7 +199,7 @@ def get_field_summary(field_name, df): return f"{field_name} -- type: {df[field_name].dtype}, values: {val_str}" -def generate_data_summary(input_tables, include_data_samples=True): +def generate_data_summary(input_tables, include_data_samples=True, field_sample_size=7): input_table_names = [f'{string_to_py_varname(t["name"])}' for t in input_tables] @@ -208,7 +208,7 @@ def generate_data_summary(input_tables, include_data_samples=True): field_summaries = [] for input_data in input_tables: df = pd.DataFrame(input_data['rows']) - s = '\n\t'.join([get_field_summary(fname, df) for fname in list(df.columns.values)]) + s = '\n\t'.join([get_field_summary(fname, df, field_sample_size) for fname in list(df.columns.values)]) field_summaries.append(s) table_field_summaries = [f'table_{i} ({input_table_names[i]}) fields:\n\t{s}' for i, s in enumerate(field_summaries)] diff --git a/src/app/App.tsx b/src/app/App.tsx index c8c5f79..ee213cb 100644 --- a/src/app/App.tsx +++ b/src/app/App.tsx @@ -142,7 +142,7 @@ export const ExportStateButton: React.FC<{}> = ({}) => { //type AppProps = ConnectedProps; -export const toolName = "Data Formulator" +export const toolName = "Data Anvil" export interface AppFCProps { } diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index 2ba7beb..564a10d 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -428,6 +428,12 @@ export const dataFormulatorSlice = createSlice({ let encoding = (state.charts.find(chart => chart.id == chartId) as Chart).encodingMap[channel]; if (prop == 'fieldID') { encoding.fieldID = value; + + // automatcially fetch the auto-sort order from the field + let field = state.conceptShelfItems.find(f => f.id == value); + if (field?.levels) { + encoding.sortBy = JSON.stringify(field.levels); + } } else if (prop == 'bin') { encoding.bin = value; } else if (prop == 'aggregate') { @@ -453,8 +459,8 @@ export const dataFormulatorSlice = createSlice({ let enc1 = chart.encodingMap[channel1]; let enc2 = chart.encodingMap[channel2]; - chart.encodingMap[channel1] = { fieldID: enc2.fieldID, aggregate: enc2.aggregate, bin: enc2.bin }; - chart.encodingMap[channel2] = { fieldID: enc1.fieldID, aggregate: enc1.aggregate, bin: enc1.bin }; + chart.encodingMap[channel1] = { fieldID: enc2.fieldID, aggregate: enc2.aggregate, bin: enc2.bin, sortBy: enc2.sortBy }; + chart.encodingMap[channel2] = { fieldID: enc1.fieldID, aggregate: enc1.aggregate, bin: enc1.bin, sortBy: enc1.sortBy }; } }, addConceptItems: (state, action: PayloadAction) => { @@ -610,6 +616,9 @@ export const dataFormulatorSlice = createSlice({ if (((field.source == "original" && field.tableRef == tableId ) || field.source == "custom") && Object.keys(typeMap).includes(field.name)) { field.semanticType = typeMap[field.name]['semantic_type']; field.type = typeMap[field.name]['type'] as Type; + if (typeMap[field.name]['sort_order']) { + field.levels = { "values": typeMap[field.name]['sort_order'], "reason": "natural sort order"} + } return field; } else { return field; diff --git a/src/app/utils.tsx b/src/app/utils.tsx index c2a41ea..59a36f7 100644 --- a/src/app/utils.tsx +++ b/src/app/utils.tsx @@ -423,7 +423,7 @@ export const instantiateVegaTemplate = (chartType: string, encodingMap: { [key i // use post processor to handle smart chart instantiation if (chartTemplate.postProcessor) { - vgObj = chartTemplate.postProcessor(vgObj); + vgObj = chartTemplate.postProcessor(vgObj, workingTable); } // console.log(JSON.stringify(vgObj)) diff --git a/src/components/ChartTemplates.tsx b/src/components/ChartTemplates.tsx index 31ee503..a3b16d8 100644 --- a/src/components/ChartTemplates.tsx +++ b/src/components/ChartTemplates.tsx @@ -158,7 +158,7 @@ const scatterPlots: ChartTemplate[] = [ "y": ["encoding", "y"], // a object can have multiple destinations "color": ["layer", 1, "encoding", "color"] }, - "postProcessor": (vgSpec: any) => { + "postProcessor": (vgSpec: any, table: any[]) => { if (vgSpec.encoding.y?.type == "nominal") { vgSpec['layer'][0]['encoding']['detail'] = JSON.parse(JSON.stringify(vgSpec['encoding']['y'])) } else if (vgSpec.encoding.x?.type == "nominal") { @@ -178,7 +178,7 @@ const scatterPlots: ChartTemplate[] = [ }, "channels": ["x", "y", "color", "opacity", "column", "row"], "paths": Object.fromEntries(["x", "y", "color", "opacity", "column", "row"].map(channel => [channel, ["encoding", channel]])), - "postProcessor": (vgSpec: any) => { + "postProcessor": (vgSpec: any, table: any[]) => { if (vgSpec.encoding.x && vgSpec.encoding.x.type != "nominal") { vgSpec.encoding.x.type = "nominal"; } @@ -204,6 +204,62 @@ const barCharts: ChartTemplate[] = [ "row": ["encoding", "row"] } }, + { + "chart": "Pyramid Chart", + "icon": chartIconColumn, + "template": { + "spacing": 0, + + "resolve": {"scale": {"y": "shared"}}, + "hconcat": [{ + "mark": "bar", + "encoding": { + "y": { }, + "x": { "scale": {"reverse": true}, "stack": null}, + "color": { "legend": null }, + "opacity": {"value": 0.9} + } + }, { + "mark": "bar", + "encoding": { + "y": {"axis": null}, + "x": {"stack": null}, + "color": { "legend": null}, + "opacity": {"value": 0.9}, + } + }], + "config": { + "view": {"stroke": null}, + "axis": {"grid": false} + }, + }, + "channels": ["x", "y", "color"], + "paths": { + "x": [["hconcat", 0, "encoding", "x"], ["hconcat", 1, "encoding", "x"]], + "y": [["hconcat", 0, "encoding", "y"], ["hconcat", 1, "encoding", "y"]], + "color": [["hconcat", 0, "encoding", "color"], ["hconcat", 1, "encoding", "color"]], + }, + "postProcessor": (vgSpec: any, table: any[]) => { + try { + if (table) { + let colorField = vgSpec['hconcat'][0]['encoding']['color']['field']; + let colorValues = [...new Set(table.map(r => r[colorField]))] ; + vgSpec.hconcat[0].transform = [{"filter": `datum[\"${colorField}\"] == \"${colorValues[0]}\"`}] + vgSpec.hconcat[0].title = colorValues[0] + vgSpec.hconcat[1].transform = [{"filter": `datum[\"${colorField}\"] == \"${colorValues[1]}\"`}] + vgSpec.hconcat[1].title = colorValues[1] + let xField = vgSpec['hconcat'][0]['encoding']['x']['field']; + let xValues = [...new Set(table.filter(r => r[colorField] == colorValues[0] || r[colorField] == colorValues[1]).map(r => r[xField]))]; + let domain = [Math.min(...xValues, 0), Math.max(...xValues)] + vgSpec.hconcat[0]['encoding']['x']['scale']['domain'] = domain; + vgSpec.hconcat[1]['encoding']['x']['scale'] = {domain: domain}; + } + } catch { + + } + return vgSpec; + } + }, { "chart": "Grouped Bar Chart", "icon": chartIconColumnGrouped, @@ -357,7 +413,7 @@ let tableCharts : ChartTemplate[] = [ }, "channels": ["x", "y", "color", "column", "row"], "paths": Object.fromEntries(["x", "y", "color", "column", "row"].map(channel => [channel, ["encoding", channel]])), - "postProcessor": (vgSpec: any) => { + "postProcessor": (vgSpec: any, table: any[]) => { if (vgSpec.encoding.y && vgSpec.encoding.y.type != "nominal") { vgSpec.encoding.y.type = "nominal"; } diff --git a/src/components/ComponentType.tsx b/src/components/ComponentType.tsx index 66f83e0..a0a6995 100644 --- a/src/components/ComponentType.tsx +++ b/src/components/ComponentType.tsx @@ -128,7 +128,7 @@ export type ChartTemplate = { template: any, channels: string[], paths: { [key: string]: (string | number)[] | (string | number)[][]; }, - postProcessor?: (vgSpec: any) => any + postProcessor?: (vgSpec: any, table: any[]) => any } export const AGGR_OP_LIST = ["count", "sum", "average"] as const diff --git a/src/views/EncodingBox.tsx b/src/views/EncodingBox.tsx index 59f4ea6..8e5bc7d 100644 --- a/src/views/EncodingBox.tsx +++ b/src/views/EncodingBox.tsx @@ -154,8 +154,13 @@ export const EncodingBox: FC = function EncodingBox({ channel, dispatch(dfActions.swapChartEncoding({chartId, channel1, channel2})) } - let handleUpdateEncoding = (channel: Channel, encoding: EncodingItem) => { - dispatch(dfActions.updateChartEncoding({chartId, channel, encoding})); + let handleResetEncoding = () => { + dispatch(dfActions.updateChartEncoding({chartId, channel, encoding: {bin: false}})); + } + + // updating a property of the encoding + let updateEncProp = (prop: keyof EncodingItem, value: any) => { + dispatch(dfActions.updateChartEncodingProp({chartId, channel, prop: prop as string, value})); } const conceptShelfItems = useSelector((state: DataFormulatorState) => state.conceptShelfItems); @@ -167,7 +172,12 @@ export const EncodingBox: FC = function EncodingBox({ channel, const dispatch = useDispatch(); - useEffect(() => { setAutoSortResult(field?.levels) }, [encoding.fieldID, field]) + useEffect(() => { + setAutoSortResult(field?.levels); + if (field?.levels) { + updateEncProp('sortBy', JSON.stringify(field?.levels)); + } + }, [encoding.fieldID, field]) // make this a drop element for concepts const [{ canDrop, isOver }, drop] = useDrop(() => ({ @@ -175,7 +185,10 @@ export const EncodingBox: FC = function EncodingBox({ channel, drop: (item: any): EncodingDropResult => { if (item.type === "concept-card") { if (item.source === "conceptShelf") { - handleUpdateEncoding(channel, { 'fieldID': item.fieldID, bin: false }); + handleResetEncoding(); + updateEncProp('fieldID', item.fieldID); + updateEncProp('bin', false); + //handleUpdateEncoding(channel, { 'fieldID': item.fieldID, bin: false }); } else if (item.source === "encodingShelf") { handleSwapEncodingField(channel, item.channel); } else { @@ -204,10 +217,6 @@ export const EncodingBox: FC = function EncodingBox({ channel, // items that control the editor panel popover const [editMode, setEditMode] = React.useState(false); - // updating a property of the encoding - let updateEncProp = (prop: keyof EncodingItem, value: any) => { - dispatch(dfActions.updateChartEncodingProp({chartId, channel, prop: prop as string, value})); - } const isActive = canDrop && isOver; let backgroundColor = ''; @@ -219,7 +228,7 @@ export const EncodingBox: FC = function EncodingBox({ channel, let fieldComponent = field === undefined ? "" : ( { - handleUpdateEncoding(channel, { 'bin': false }); + handleResetEncoding(); }} /> ) @@ -231,31 +240,6 @@ export const EncodingBox: FC = function EncodingBox({ channel, value={value} control={} label={label} /> } - let aggrOpt = [ - Aggregate, - - { updateEncProp("aggregate", event.target.value == "none" ? undefined : event.target.value as AggrOp); }} - > - {radioLabel("none", "none", `aggr--1`)} - {AGGR_OP_LIST.map((t, i) => radioLabel(t, t, `aggr-${i}`))} - - - ] - let stackOpt = (chart.chartType == "bar" || chart.chartType == "area") && (channel == "x" || channel == "y") ? [ Stack, = function EncodingBox({ channel, ] : []; - let binOpt = [ - Bin, - - { updateEncProp("bin", event.target.value == "on"); }} - > - {radioLabel("off", "off", `bin-radio-off`)} - {radioLabel("on", "on", `bin-radio-on`)} - - - ] - let domainItems = (field?.source == "custom" || field?.source == "original") ? getDomains(field as FieldItem, tables)[0] : []; if (field?.source == "derived") { domainItems = deriveTransformExamplesV2( @@ -374,129 +335,6 @@ export const EncodingBox: FC = function EncodingBox({ channel, }); } - let autoSortBtn = - - let sortOptions = [radioLabel("↑ asc", "ascending", `sort-ascending`), radioLabel("↓ desc", "descending", `sort-descending`)] - let extraSortOptions = []; - - // TODO: check sort options - if (channel == "x" && (field?.type == "string" || field?.type == "auto")) { - - extraSortOptions.push(radioLabel("y↑ asc", "y", `sort-x-y-ascending`, 90)); - extraSortOptions.push(radioLabel("y↓ desc", "-y", `sort-x-y-descending`, 90)) - } - if (channel == "y" && (field?.type == "string" || field?.type == "auto")) { - extraSortOptions.push(radioLabel("x↑", "x", `sort-y-x-ascending`, 90)); - extraSortOptions.push(radioLabel("x↓", "-x", `sort-y-x-descending`, 90)) - } - if (extraSortOptions.length > 0) { - sortOptions = [ - radioLabel("↑ asc", "ascending", `sort-ascending`, 90), - radioLabel("↓ desc", "descending", `sort-descending`, 90), - ...extraSortOptions]; - } - - // if (autoSortEnabled) { - // if (autoSortResult != undefined && autoSortResult.length > 0) { - // let autoSortOpt = - // - // {autoSortResult.map(x => x ? x.toString() : 'null').join(", ")} - // ; - - // let autoSortOptReversed = - // - // {[...autoSortResult].reverse().map(x =>x ? x.toString() : 'null' ).join(", ")} - // ; - - // sortOptions = [ - // ...sortOptions, - // } - // label={autoSortOpt} />, - // } - // label={autoSortOptReversed} />, - // } - // label={autoSortBtn} /> - // ] - // } else { - // sortOptions = [ - // ...sortOptions, - // } - // label={autoSortBtn} /> - // ] - // } - // } - - let sortByFieldInputBox = { - console.log(`change: ${value}`) - }} - // value={tempValue} - filterOptions={(options, params) => { - const filtered = filter(options, params); - const { inputValue } = params; - // Suggest the creation of a new value - const isExisting = options.some((option) => inputValue === option); - if (inputValue !== '' && !isExisting) { - return [...filtered, `${inputValue}`, ] - } else { - return [...filtered]; - } - }} - sx={{ flexGrow: 1, flexShrink: 1, width: '120px', "& .MuiInput-input": { padding: "0px 8px !important"}}} - fullWidth - selectOnFocus - clearOnBlur - handleHomeEndKeys - autoHighlight - id="free-solo-with-text-demo" - options={conceptShelfItems.map(f => f.name).filter(name => name != "")} - getOptionLabel={(option) => { - // Value selected with enter, right from the input - return option; - }} - groupBy={(option) => { - let groupItem = conceptGroups.find(item => item.field.name == option); - if (groupItem && groupItem.field.name != "") { - return `from ${groupItem.group}`; - } else { - return "create a new concept" - } - }} - renderGroup={(params) => ( -
  • - {params.group} - {params.children} -
    • - )} - renderOption={(props, option) => { - let renderOption = (conceptShelfItems.map(f => f.name).includes(option) || option == "...") ? option : `"${option}"`; - let otherStyle = option == `...` ? {color: "darkgray"} : {} - - return { - //handleSelectOption(option); - }} sx={{fontSize: "small", ...otherStyle}}>{renderOption} - }} - freeSolo - renderInput={(params) => ( - - )} - /> - let sortByFieldID = encoding.fieldID - let sortByOptions = [ radioLabel("default", "default", `sort-by-default`) ] @@ -508,14 +346,6 @@ export const EncodingBox: FC = function EncodingBox({ channel, sortByOptions.push(radioLabel("x values", "x", `sort-y-by-x-ascending`, 90)); } - // ***** sort by field option ***** - // sortByOptions = [ - // ...sortByOptions, - // } - // label={sortByFieldInputBox} /> - // ] - if (autoSortEnabled) { if (autoSortInferRunning) { sortByOptions = [ @@ -528,7 +358,7 @@ export const EncodingBox: FC = function EncodingBox({ channel, } else { if (autoSortResult != undefined) { - let autoSortOptTitle = + let autoSortOptTitle = Sort Order: {autoSortResult.values.map(x => x ? x.toString() : 'null').join(", ")} @@ -594,7 +424,7 @@ export const EncodingBox: FC = function EncodingBox({ channel, row aria-labelledby="sort-option-radio-buttons-group" name="sort-option-radio-buttons-group" - value={encoding.sortBy || 'default'} + value={encoding.sortBy || 'default'} sx={{ width: 180 }} onChange={(event) => { updateEncProp("sortBy", event.target.value) }} > @@ -625,28 +455,6 @@ export const EncodingBox: FC = function EncodingBox({ channel, ] - - - // let sortOpt = [ - // Sort By, - // - // { updateEncProp("sort", event.target.value) }} - // > - // {sortOptions} - // - // , - // ] let colorSchemeList = [ "category10", @@ -688,48 +496,6 @@ export const EncodingBox: FC = function EncodingBox({ channel, margin: '0px 12px', padding: "6px", fontSize: "12px" }} > - {/* :not(style)': { margin: "0px", }, }} - noValidate - autoComplete="off"> - - Data Field - - - */} - {/* :not(style)': { margin: "4px", }, }} - noValidate - autoComplete="off"> - - Aggregate - - - */} - {/* {aggrOpt} - {binOpt} */} {stackOpt} {sortByOpt} {sortOrderOpt} From 24ba6bd2ff0f0b931f14593bfcb1a33ad0bb6db0 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Wed, 2 Oct 2024 17:10:33 -0700 Subject: [PATCH 02/39] experimental data cleaning on load function --- app.py | 32 ++- package.json | 1 + server/agents/__init__.py | 2 + server/agents/agent_data_clean.py | 142 +++++++++++++ src/app/App.tsx | 2 +- src/app/utils.tsx | 2 +- src/scss/EncodingShelf.scss | 8 +- src/views/DataFormulator.tsx | 4 +- src/views/EncodingBox.tsx | 2 +- src/views/EncodingShelfCard.tsx | 2 +- src/views/TableSelectionView.tsx | 275 +++++++++++++++++++++++- yarn.lock | 336 +++++++++++++++++++++++++++++- 12 files changed, 787 insertions(+), 21 deletions(-) create mode 100644 server/agents/agent_data_clean.py diff --git a/app.py b/app.py index 57b4f1c..4707c6d 100644 --- a/app.py +++ b/app.py @@ -22,14 +22,12 @@ sys.path.append(os.path.abspath(APP_ROOT)) from agents.agent_concept_derive import ConceptDeriveAgent -from agents.agent_data_transformation import DataTransformationAgent from agents.agent_data_transform_v2 import DataTransformationAgentV2 from agents.agent_data_rec import DataRecAgent from agents.agent_sort_data import SortDataAgent from agents.agent_data_load import DataLoadAgent -from agents.agent_data_filter import DataFilterAgent -from agents.agent_generic_py_concept import GenericPyConceptDeriveAgent +from agents.agent_data_clean import DataCleanAgent from agents.agent_code_explanation import CodeExplanationAgent from agents.client_utils import get_client @@ -273,6 +271,34 @@ def derive_concept_request(): response.headers.add('Access-Control-Allow-Origin', '*') return response + +@app.route('/clean-data', methods=['GET', 'POST']) +def clean_data_request(): + + if request.is_json: + app.logger.info("# data clean request") + content = request.get_json() + token = content["token"] + + client = get_client(content['model']['endpoint'], content['model']['key']) + model = content['model']['model'] + + app.logger.info(f" model: {content['model']}") + + agent = DataCleanAgent(client=client, model=model) + + candidates = agent.run(content["raw_data"]) + + candidates = [c for c in candidates if c['status'] == 'ok'] + + response = flask.jsonify({ "status": "ok", "token": token, "result": candidates }) + else: + response = flask.jsonify({ "token": -1, "status": "error", "result": [] }) + + response.headers.add('Access-Control-Allow-Origin', '*') + return response + + @app.route('/codex-sort-request', methods=['GET', 'POST']) def sort_data_request(): diff --git a/package.json b/package.json index f6edbba..4f714ad 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "react": "^18.2.0", "react-animate-height": "^3.0.4", "react-animate-on-change": "^2.2.0", + "react-diff-viewer": "^3.1.1", "react-dnd": "^16.0.1", "react-dnd-html5-backend": "^16.0.1", "react-dom": "^18.2.0", diff --git a/server/agents/__init__.py b/server/agents/__init__.py index 78fa41a..6b8fc97 100644 --- a/server/agents/__init__.py +++ b/server/agents/__init__.py @@ -7,6 +7,7 @@ from .agent_data_transform_v2 import DataTransformationAgentV2 from .agent_data_load import DataLoadAgent from .agent_sort_data import SortDataAgent +from .agent_data_clean import DataCleanAgent from agents.agent_data_rec import DataRecAgent __all__ = [ @@ -17,4 +18,5 @@ "DataRecAgent", "DataLoadAgent", "SortDataAgent", + "DataCleanAgent" ] \ No newline at end of file diff --git a/server/agents/agent_data_clean.py b/server/agents/agent_data_clean.py new file mode 100644 index 0000000..34ab4f4 --- /dev/null +++ b/server/agents/agent_data_clean.py @@ -0,0 +1,142 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json +import pandas as pd + +from agents.agent_utils import extract_json_objects, generate_data_summary, extract_code_from_gpt_response, field_name_to_ts_variable_name, infer_ts_datatype +import py_sandbox +import traceback + +import logging + +logger = logging.getLogger(__name__) + + +SYSTEM_PROMPT = '''You are a data scientist to help user to generate or clean the raw input into a *csv block* (or tsv if that's the original format). +The output csv format should be readable into a python pandas dataframe directly. + +Create [OUTPUT] based on [RAW DATA] provided. The output should have two components: + +1. a csv codeblock that represents the cleaned data, as follows: + +```csv +..... +``` + +2. a json object that explains the mode and cleaning rationale (wrap in a json block): + +```json +{ + "mode": ..., // one of "data generation" or "data cleaning" based on the provided task + "reason": ... // explain the cleaning reason here +} +``` + +**Important:** +- NEVER make assumptions or judgments about a person's gender, biological sex, sexuality, religion, race, nationality, ethnicity, political stance, socioeconomic status, mental health, invisible disabilities, medical conditions, personality type, social impressions, emotional state, and cognitive state. +- NEVER create formulas that could be used to discriminate based on age. Ageism of any form (explicit and implicit) is strictly prohibited. +- If above issue occurs, just copy the original data and return in the block + +The cleaning process must follow instructions below: +* the output should be a structured csv table: + - if the raw data is unstructured, structure it into a csv table. If the table is in other formats, transform it into a csv table. + - if the raw data contain other informations other than the table, remove surrounding texts that does not belong to the table. + - if the raw data contains multiple levels of header, make it a flat table. It's ok to combine multiple levels of headers to form the new header to not lose information. + - if the table has footer or summary row, remove them, since they would not be compatible with the csv table format. + - the csv table should have the same number of cells for each line, according to the title. If there are some rows with missing values, patch them with empty cells. + - if the raw data has some rows that do not belong to the table, also remove them (e.g., subtitles in between rows) + - if the header row misses some columns, add their corresponding column names. E.g., when the header doesn't have an index column, but every row has an index value, add the missing column header. +* clean up columns with messy information + - if a column is number but some cells has annotations like "*" "?" or brackets, clean them up. + - you don't need to convert format of the cell. +* if the user asks about generating synthetic data: + - NEVER generate data that has implicit bias as noted above, if that happens, return a dummy data consisting of dummy columns with 'a, b, c' and numbers. + - NEVER generate data contain people's names, use "A" , "B", "C"... instead. + - If the user doesn't indicate how many rows to be generated, plan in generating a dataset with 10-20 rows depending on the content. +''' + +EXAMPLE = ''' +[RAW DATA] + +Dates Samplei Pollster Sponsor Result Net result +Polls ending today +President: general election, 2024Icon indicating this set of polls has an average. +AVG. +Sept. 22-26 +1,735 LV +Outward Intelligence +Harris 51% 45% Trump More Harris +6 +Sept. 22-26 +1,735 LV +Outward Intelligence +Harris 53% 47% Trump Harris +6 +Sept. 25, 2024 +Joe Biden ApprovalIcon indicating this set of polls has an average. +AVG. +Sept. 23-25 +1,005 LV +Echelon Insights +Approve 43% 55% Disapprove Disapprove +12 +Sept. 23-25 +1,524 LV +Big Village +Approve 46% 52% Disapprove Disapprove +6 +Sept. 23-25 +1,663 RV +Big Village +Approve 46% 51% Disapprove Disapprove +5 +Sept. 23-25 +2,021 A +Big Village +Approve 44% 50% Disapprove Disapprove +6 + +[OUTPUT] + +''' + +class DataCleanAgent(object): + + def __init__(self, client, model): + self.model = model + self.client = client + + def run(self, raw_data): + """derive a new concept based on the raw input data + """ + + user_query = f"[DATA]\n\n{raw_data}\n\n[OUTPUT]\n" + + logger.info(user_query) + + messages = [{"role":"system", "content": SYSTEM_PROMPT}, + {"role":"user","content": user_query}] + + ###### the part that calls open_ai + response = self.client.chat.completions.create( + model=self.model, messages = messages, temperature=0.7, max_tokens=1200, + top_p=0.95, n=1, frequency_penalty=0, presence_penalty=0, stop=None) + + candidates = [] + for choice in response.choices: + + logger.info("\n=== Python Data Clean Agent ===>\n") + logger.info(choice.message.content + "\n") + + code_blocks = extract_code_from_gpt_response(choice.message.content + "\n", "csv") + reason_blocks = extract_json_objects(choice.message.content + "\n") + + if len(code_blocks) > 0: + result = { + 'status': 'ok', + 'content': code_blocks[-1], + 'info': reason_blocks[-1] if len(reason_blocks) > 0 else {"reason": "no reason presented", "mode": "data cleaning"} + } + else: + result = {'status': 'other error', 'content': 'unable to extract code from response'} + + result['dialog'] = [*messages, {"role": choice.message.role, "content": choice.message.content}] + result['agent'] = 'DataCleanAgent' + candidates.append(result) + + return candidates \ No newline at end of file diff --git a/src/app/App.tsx b/src/app/App.tsx index ee213cb..c8c5f79 100644 --- a/src/app/App.tsx +++ b/src/app/App.tsx @@ -142,7 +142,7 @@ export const ExportStateButton: React.FC<{}> = ({}) => { //type AppProps = ConnectedProps; -export const toolName = "Data Anvil" +export const toolName = "Data Formulator" export interface AppFCProps { } diff --git a/src/app/utils.tsx b/src/app/utils.tsx index 59a36f7..a815fa0 100644 --- a/src/app/utils.tsx +++ b/src/app/utils.tsx @@ -37,8 +37,8 @@ export function getUrls() { // these functions involves openai models DERIVE_CONCEPT_URL: `${appConfig.serverUrl}/derive-concept-request`, SORT_DATA_URL: `${appConfig.serverUrl}/codex-sort-request`, + CLEAN_DATA_URL: `${appConfig.serverUrl}/clean-data`, SERVER_DERIVE_DATA_URL: `${appConfig.serverUrl}/derive-data`, - SERVER_DERIVE_DATA_V2_URL: `${appConfig.serverUrl}/dispatch-and-derive-data`, SERVER_REFINE_DATA_URL: `${appConfig.serverUrl}/refine-data`, CODE_EXPL_URL: `${appConfig.serverUrl}/code-expl`, SERVER_PROCESS_DATA_ON_LOAD: `${appConfig.serverUrl}/process-data-on-load`, diff --git a/src/scss/EncodingShelf.scss b/src/scss/EncodingShelf.scss index 163203e..c20f32e 100644 --- a/src/scss/EncodingShelf.scss +++ b/src/scss/EncodingShelf.scss @@ -46,19 +46,17 @@ } } - .auto-sort-option-label { width: calc(100% - 24px); overflow: hidden; display: -webkit-box; - -webkit-line-clamp: 1; + line-clamp: 2; + -webkit-line-clamp: 2; -webkit-box-orient: vertical; background: #fff; text-align: left; - font-size: smaller; white-space: normal; - font-size: smaller !important; - font-style: italic; + font-size: xx-small !important; } .channel-shelf-box { diff --git a/src/views/DataFormulator.tsx b/src/views/DataFormulator.tsx index 1241a85..a2da0cb 100644 --- a/src/views/DataFormulator.tsx +++ b/src/views/DataFormulator.tsx @@ -31,7 +31,7 @@ import { DndProvider } from 'react-dnd' import { HTML5Backend } from 'react-dnd-html5-backend' import { SelectableGroup } from 'react-selectable-fast'; -import { TableSelectionDialog, TableURLDialog } from './TableSelectionView'; +import { TableCopyDialogV2, TableSelectionDialog, TableURLDialog } from './TableSelectionView'; import { TableCopyDialog, TableUploadDialog } from './TableSelectionView'; import { toolName } from '../app/App'; import { DataThread } from './DataThread'; @@ -128,7 +128,7 @@ export const DataFormulatorFC = ({ }) => { Load data from - , , , or + , , or Select or upload datasets (in csv, tsv or json records format) to get started. diff --git a/src/views/EncodingBox.tsx b/src/views/EncodingBox.tsx index 8e5bc7d..0d944c8 100644 --- a/src/views/EncodingBox.tsx +++ b/src/views/EncodingBox.tsx @@ -406,7 +406,7 @@ export const EncodingBox: FC = function EncodingBox({ channel, value={JSON.stringify(autoSortResult)} control={} label={} /> + onClick={autoSortFunction}>infer smart sort order} /> ] } } diff --git a/src/views/EncodingShelfCard.tsx b/src/views/EncodingShelfCard.tsx index b31ebb0..cb238de 100644 --- a/src/views/EncodingShelfCard.tsx +++ b/src/views/EncodingShelfCard.tsx @@ -275,7 +275,7 @@ export const EncodingShelfCard: FC = function ({ chartId extra_prompt: instruction, model: activeModel }) - let engine = betaMode ? getUrls().SERVER_DERIVE_DATA_V2_URL : getUrls().SERVER_DERIVE_DATA_URL; + let engine = getUrls().SERVER_DERIVE_DATA_URL; if (mode == "formulate" && currentTable.derive?.dialog) { messageBody = JSON.stringify({ diff --git a/src/views/TableSelectionView.tsx b/src/views/TableSelectionView.tsx index 20ae0eb..fab76c8 100644 --- a/src/views/TableSelectionView.tsx +++ b/src/views/TableSelectionView.tsx @@ -7,7 +7,8 @@ import Tabs from '@mui/material/Tabs'; import Tab from '@mui/material/Tab'; import Typography from '@mui/material/Typography'; import Box from '@mui/material/Box'; -import { Button, Dialog, DialogActions, DialogContent, DialogTitle, IconButton, Input, Paper, TextField } from '@mui/material'; +import { alpha, Button, Collapse, Dialog, DialogActions, DialogContent, DialogTitle, Divider, + IconButton, Input, CircularProgress, LinearProgress, Paper, TextField, useTheme } from '@mui/material'; import { CustomReactTable } from './ReactTable'; import { DictTable } from "../components/ComponentType"; @@ -16,8 +17,16 @@ import { getUrls } from '../app/utils'; import { createTableFromFromObjectArray, createTableFromText, loadDataWrapper } from '../data/utils'; import CloseIcon from '@mui/icons-material/Close'; -import { dfActions, fetchFieldSemanticType } from '../app/dfSlice'; -import { useDispatch } from 'react-redux'; +import KeyboardReturnIcon from '@mui/icons-material/KeyboardReturn'; +import ChevronLeftIcon from '@mui/icons-material/ChevronLeft'; +import ChevronRightIcon from '@mui/icons-material/ChevronRight'; +import AutoFixNormalIcon from '@mui/icons-material/AutoFixNormal'; +import OpenInNewIcon from '@mui/icons-material/OpenInNew'; + +import ReactDiffViewer from 'react-diff-viewer' + +import { dfActions, dfSelectors, fetchFieldSemanticType } from '../app/dfSlice'; +import { useDispatch, useSelector } from 'react-redux'; import { useState } from 'react'; import { AppDispatch } from '../app/store'; @@ -393,4 +402,262 @@ export const TableURLDialog: React.FC = ({ buttonElement, d {dialog} ; -} \ No newline at end of file +} + + +export const TableCopyDialogV2: React.FC = ({ buttonElement, disabled }) => { + + let activeModel = useSelector(dfSelectors.getActiveModel); + + const [dialogOpen, setDialogOpen] = useState(false); + const [tableName, setTableName] = useState(""); + const [tableContent, setTableContent] = useState(""); + + + const [cleaningInProgress, setCleaningInProgress] = useState(false); + const [cleanTableContent, setCleanTableContent] = useState<{content: string, reason: string, mode: string} | undefined>(undefined); + + let viewTable = cleanTableContent == undefined ? undefined : createTableFromText(tableName || "clean-table", cleanTableContent.content) + + + const [loadFromURL, setLoadFromURL] = useState(false); + const [url, setURL] = useState(""); + + let theme = useTheme() + + const dispatch = useDispatch(); + + let handleSubmitContent = (tableStr: string): void => { + let table : undefined | DictTable = undefined; + try { + let content = JSON.parse(tableStr); + table = createTableFromFromObjectArray(tableName || 'data-0', content); + } catch (error) { + table = createTableFromText(tableName || 'data-0', tableStr); + } + if (table) { + dispatch(dfActions.addTable(table)); + dispatch(fetchFieldSemanticType(table)); + } + }; + + let handleLoadURL = () => { + console.log("hello hello") + setLoadFromURL(!loadFromURL); + + let parts = url.split('/'); + + // Get the last part of the URL, which should be the file name with extension + const tableName = parts[parts.length - 1]; + + fetch(url) + .then(res => res.text()) + .then(content => { + setTableName(tableName); + setTableContent(content); + }) + } + + let handleCleanData = () => { + //setCleanTableContent("hehehao\n\n" + tableContent); + let token = String(Date.now()); + setCleaningInProgress(true); + setCleanTableContent(undefined); + let message = { + method: 'POST', + headers: { 'Content-Type': 'application/json', }, + body: JSON.stringify({ + token: token, + raw_data: tableContent, + model: activeModel + }), + }; + + fetch(getUrls().CLEAN_DATA_URL, message) + .then((response) => response.json()) + .then((data) => { + setCleaningInProgress(false); + console.log(data); + console.log(token); + + if (data["status"] == "ok") { + if (data["token"] == token) { + let candidate = data["result"][0]; + console.log(candidate) + + let cleanContent = candidate['content']; + let info = candidate['info']; + + setCleanTableContent({content: cleanContent.trim(), reason: info['reason'], mode: info['mode']}); + console.log(`data cleaning reason:`) + console.log(info); + } + } else { + // TODO: add warnings to show the user + dispatch(dfActions.addMessages({ + "timestamp": Date.now(), + "type": "error", + "value": "unable to perform auto-sort." + })); + setCleanTableContent(undefined); + } + }).catch((error) => { + setCleaningInProgress(false); + setCleanTableContent(undefined); + + dispatch(dfActions.addMessages({ + "timestamp": Date.now(), + "type": "error", + "value": "unable to perform clean data due to server issue." + })); + }); + } + + const newStyles = { + variables: { }, + line: { + '&:hover': { + background: alpha(theme.palette.primary.main, 0.2), + }, + }, + titleBlock: { + padding: '4px 8px', + borderBottom: 'none' + }, + marker: { + width: 'fit-content' + }, + content: { + fontSize: 12, + width: 'fit-content', + maxWidth: "50%", + minWidth: 300, + }, + diffContainer: { + "pre": { lineHeight: 1.2, fontFamily: 'sans-serif' } + }, + contentText: { + + }, + gutter: { + minWidth: '12px', + fontSize: 12, + padding: '0 8px', + } + }; + + let renderLines = (str: string) => ( + {str} + ); + + let dialog = {setDialogOpen(false)}} open={dialogOpen} + sx={{ '& .MuiDialog-paper': { maxWidth: '80%', maxHeight: 800, minWidth: 800 } }} + > + Paste & Upload Data + { setDialogOpen(false); }} + aria-label="close" + > + + + + + + { setTableName(event.target.value); }} + autoComplete='off' id="outlined-basic" label="dataset name" variant="outlined" /> + + + + + { setURL(event.target.value); }} + onKeyDown={(event)=> { + if(event.key == 'Enter'){ + handleLoadURL(); + event.preventDefault(); + } + }} + autoComplete='off' id="outlined-basic" label="url" variant="outlined" /> + + + + + + {cleaningInProgress ? : ""} + { viewTable ? + <> + {/* */} + { + return { id: name, label: name, minWidth: 60, align: undefined, format: (v: any) => v} + })} /> + {/* {cleanTableContent.reason} */} + + : + { setTableContent(event.target.value); }} + autoComplete='off' + label="content (csv, tsv, or json format)" variant="outlined" multiline minRows={15} + /> + } + + + + { cleanTableContent != undefined ? + + + + : } + {/* + + */} + + + + ; + + return <> + + {dialog} + ; +} diff --git a/yarn.lock b/yarn.lock index 16cbc91..9786a1e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9,6 +9,32 @@ dependencies: "@babel/highlight" "^7.22.5" +"@babel/code-frame@^7.24.7": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.7.tgz#882fd9e09e8ee324e496bd040401c6f046ef4465" + integrity sha512-BcYH1CVJBO9tvyIZ2jVeXgSIMvGZ2FDRvDdOIVQyuklNKSsx+eppDEBq/g47Ayw+RqNFE+URvOShmf+f/qwAlA== + dependencies: + "@babel/highlight" "^7.24.7" + picocolors "^1.0.0" + +"@babel/generator@^7.25.6": + version "7.25.6" + resolved "https://registry.npmjs.org/@babel/generator/-/generator-7.25.6.tgz#0df1ad8cb32fe4d2b01d8bf437f153d19342a87c" + integrity sha512-VPC82gr1seXOpkjAAKoLhP50vx4vGNlF4msF64dSFq1P8RfB+QAuJWGHPXXPc8QyfVWwwB/TNNU4+ayZmHNbZw== + dependencies: + "@babel/types" "^7.25.6" + "@jridgewell/gen-mapping" "^0.3.5" + "@jridgewell/trace-mapping" "^0.3.25" + jsesc "^2.5.1" + +"@babel/helper-module-imports@^7.0.0": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.24.7.tgz#f2f980392de5b84c3328fc71d38bd81bbb83042b" + integrity sha512-8AyH3C+74cgCVVXow/myrynrAGv+nTVg5vKu2nZph9x7RcRwzmh0VFallJuFTZ9mx6u4eSdXZfcOzSqTUm0HCA== + dependencies: + "@babel/traverse" "^7.24.7" + "@babel/types" "^7.24.7" + "@babel/helper-module-imports@^7.16.7": version "7.16.7" resolved "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.16.7.tgz" @@ -21,11 +47,21 @@ resolved "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw== +"@babel/helper-string-parser@^7.24.8": + version "7.24.8" + resolved "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.24.8.tgz#5b3329c9a58803d5df425e5785865881a81ca48d" + integrity sha512-pO9KhhRcuUyGnJWwyEgnRJTSIZHiT+vMD0kPeD+so0l7mxkMT19g3pjY9GTnHySck/hDzq+dtW/4VgnMkippsQ== + "@babel/helper-validator-identifier@^7.22.5": version "7.22.5" resolved "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.5.tgz#9544ef6a33999343c8740fa51350f30eeaaaf193" integrity sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ== +"@babel/helper-validator-identifier@^7.24.7": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz#75b889cfaf9e35c2aaf42cf0d72c8e91719251db" + integrity sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w== + "@babel/highlight@^7.22.5": version "7.22.5" resolved "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.5.tgz#aa6c05c5407a67ebce408162b7ede789b4d22031" @@ -35,6 +71,23 @@ chalk "^2.0.0" js-tokens "^4.0.0" +"@babel/highlight@^7.24.7": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/highlight/-/highlight-7.24.7.tgz#a05ab1df134b286558aae0ed41e6c5f731bf409d" + integrity sha512-EStJpq4OuY8xYfhGVXngigBJRWxftKX9ksiGDnmlY3o7B/V7KIAc9X4oiK87uPJSc/vs5L869bem5fhZa8caZw== + dependencies: + "@babel/helper-validator-identifier" "^7.24.7" + chalk "^2.4.2" + js-tokens "^4.0.0" + picocolors "^1.0.0" + +"@babel/parser@^7.25.0", "@babel/parser@^7.25.6": + version "7.25.6" + resolved "https://registry.npmjs.org/@babel/parser/-/parser-7.25.6.tgz#85660c5ef388cbbf6e3d2a694ee97a38f18afe2f" + integrity sha512-trGdfBdbD0l1ZPmcJ83eNxB9rbEax4ALFTF7fN386TMYbeCQbyme5cOEXQhbGXKebwGaB/J52w1mrklMcbgy6Q== + dependencies: + "@babel/types" "^7.25.6" + "@babel/runtime@^7.12.1", "@babel/runtime@^7.12.5", "@babel/runtime@^7.9.2": version "7.17.7" resolved "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.7.tgz" @@ -49,6 +102,35 @@ dependencies: regenerator-runtime "^0.13.11" +"@babel/runtime@^7.7.2": + version "7.25.6" + resolved "https://registry.npmjs.org/@babel/runtime/-/runtime-7.25.6.tgz#9afc3289f7184d8d7f98b099884c26317b9264d2" + integrity sha512-VBj9MYyDb9tuLq7yzqjgzt6Q+IBQLrGZfdjOekyEirZPHxXWoTSGUTMrpsfi58Up73d13NfYLv8HT9vmznjzhQ== + dependencies: + regenerator-runtime "^0.14.0" + +"@babel/template@^7.25.0": + version "7.25.0" + resolved "https://registry.npmjs.org/@babel/template/-/template-7.25.0.tgz#e733dc3134b4fede528c15bc95e89cb98c52592a" + integrity sha512-aOOgh1/5XzKvg1jvVz7AVrx2piJ2XBi227DHmbY6y+bM9H2FlN+IfecYu4Xl0cNiiVejlsCri89LUsbj8vJD9Q== + dependencies: + "@babel/code-frame" "^7.24.7" + "@babel/parser" "^7.25.0" + "@babel/types" "^7.25.0" + +"@babel/traverse@^7.24.7": + version "7.25.6" + resolved "https://registry.npmjs.org/@babel/traverse/-/traverse-7.25.6.tgz#04fad980e444f182ecf1520504941940a90fea41" + integrity sha512-9Vrcx5ZW6UwK5tvqsj0nGpp/XzqthkT0dqIc9g1AdtygFToNtTF67XzYS//dm+SAK9cp3B9R4ZO/46p63SCjlQ== + dependencies: + "@babel/code-frame" "^7.24.7" + "@babel/generator" "^7.25.6" + "@babel/parser" "^7.25.6" + "@babel/template" "^7.25.0" + "@babel/types" "^7.25.6" + debug "^4.3.1" + globals "^11.1.0" + "@babel/types@^7.16.7": version "7.22.5" resolved "https://registry.npmjs.org/@babel/types/-/types-7.22.5.tgz#cd93eeaab025880a3a47ec881f4b096a5b786fbe" @@ -58,6 +140,15 @@ "@babel/helper-validator-identifier" "^7.22.5" to-fast-properties "^2.0.0" +"@babel/types@^7.24.7", "@babel/types@^7.25.0", "@babel/types@^7.25.6": + version "7.25.6" + resolved "https://registry.npmjs.org/@babel/types/-/types-7.25.6.tgz#893942ddb858f32ae7a004ec9d3a76b3463ef8e6" + integrity sha512-/l42B1qxpG6RdfYf343Uw1vmDjeNhneUXtzhojE7pDgfpEypmRhI6j1kr17XCVv4Cgl9HdAiQY2x0GwKm7rWCw== + dependencies: + "@babel/helper-string-parser" "^7.24.8" + "@babel/helper-validator-identifier" "^7.24.7" + to-fast-properties "^2.0.0" + "@emotion/babel-plugin@^11.11.0": version "11.11.0" resolved "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.11.0.tgz#c2d872b6a7767a9d176d007f5b31f7d504bb5d6c" @@ -75,6 +166,16 @@ source-map "^0.5.7" stylis "4.2.0" +"@emotion/cache@^10.0.27": + version "10.0.29" + resolved "https://registry.npmjs.org/@emotion/cache/-/cache-10.0.29.tgz#87e7e64f412c060102d589fe7c6dc042e6f9d1e0" + integrity sha512-fU2VtSVlHiF27empSbxi1O2JFdNWZO+2NFHfwO0pxgTep6Xa3uGb+3pVKfLww2l/IBGLNEZl5Xf/++A4wAYDYQ== + dependencies: + "@emotion/sheet" "0.9.4" + "@emotion/stylis" "0.8.5" + "@emotion/utils" "0.11.3" + "@emotion/weak-memoize" "0.2.5" + "@emotion/cache@^11.11.0": version "11.11.0" resolved "https://registry.npmjs.org/@emotion/cache/-/cache-11.11.0.tgz#809b33ee6b1cb1a625fef7a45bc568ccd9b8f3ff" @@ -86,6 +187,11 @@ "@emotion/weak-memoize" "^0.3.1" stylis "4.2.0" +"@emotion/hash@0.8.0": + version "0.8.0" + resolved "https://registry.npmjs.org/@emotion/hash/-/hash-0.8.0.tgz#bbbff68978fefdbe68ccb533bc8cbe1d1afb5413" + integrity sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow== + "@emotion/hash@^0.9.1": version "0.9.1" resolved "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.1.tgz#4ffb0055f7ef676ebc3a5a91fb621393294e2f43" @@ -98,6 +204,11 @@ dependencies: "@emotion/memoize" "^0.8.1" +"@emotion/memoize@0.7.4": + version "0.7.4" + resolved "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.7.4.tgz#19bf0f5af19149111c40d98bb0cf82119f5d9eeb" + integrity sha512-Ja/Vfqe3HpuzRsG1oBtWTHk2PGZ7GR+2Vz5iYGelAw8dx32K0y7PjVuxK6z1nMpZOqAFsRUPCkK1YjJ56qJlgw== + "@emotion/memoize@^0.8.1": version "0.8.1" resolved "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.8.1.tgz#c1ddb040429c6d21d38cc945fe75c818cfb68e17" @@ -117,6 +228,17 @@ "@emotion/weak-memoize" "^0.3.1" hoist-non-react-statics "^3.3.1" +"@emotion/serialize@^0.11.15", "@emotion/serialize@^0.11.16": + version "0.11.16" + resolved "https://registry.npmjs.org/@emotion/serialize/-/serialize-0.11.16.tgz#dee05f9e96ad2fb25a5206b6d759b2d1ed3379ad" + integrity sha512-G3J4o8by0VRrO+PFeSc3js2myYNOXVJ3Ya+RGVxnshRYgsvErfAOglKAiy1Eo1vhzxqtUvjCyS5gtewzkmvSSg== + dependencies: + "@emotion/hash" "0.8.0" + "@emotion/memoize" "0.7.4" + "@emotion/unitless" "0.7.5" + "@emotion/utils" "0.11.3" + csstype "^2.5.7" + "@emotion/serialize@^1.1.2": version "1.1.2" resolved "https://registry.npmjs.org/@emotion/serialize/-/serialize-1.1.2.tgz#017a6e4c9b8a803bd576ff3d52a0ea6fa5a62b51" @@ -128,6 +250,11 @@ "@emotion/utils" "^1.2.1" csstype "^3.0.2" +"@emotion/sheet@0.9.4": + version "0.9.4" + resolved "https://registry.npmjs.org/@emotion/sheet/-/sheet-0.9.4.tgz#894374bea39ec30f489bbfc3438192b9774d32e5" + integrity sha512-zM9PFmgVSqBw4zL101Q0HrBVTGmpAxFZH/pYx/cjJT5advXguvcgjHFTCaIO3enL/xr89vK2bh0Mfyj9aa0ANA== + "@emotion/sheet@^1.2.2": version "1.2.2" resolved "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.2.2.tgz#d58e788ee27267a14342303e1abb3d508b6d0fec" @@ -145,6 +272,16 @@ "@emotion/use-insertion-effect-with-fallbacks" "^1.0.1" "@emotion/utils" "^1.2.1" +"@emotion/stylis@0.8.5": + version "0.8.5" + resolved "https://registry.npmjs.org/@emotion/stylis/-/stylis-0.8.5.tgz#deacb389bd6ee77d1e7fcaccce9e16c5c7e78e04" + integrity sha512-h6KtPihKFn3T9fuIrwvXXUOwlx3rfUvfZIcP5a6rh8Y7zjE3O06hT5Ss4S/YI1AYhuZ1kjaE/5EaOOI2NqSylQ== + +"@emotion/unitless@0.7.5": + version "0.7.5" + resolved "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.7.5.tgz#77211291c1900a700b8a78cfafda3160d76949ed" + integrity sha512-OWORNpfjMsSSUBVrRBVGECkhWcULOAJz9ZW8uK9qgxD+87M7jHRcvh/A96XXNhXTLmKcoYSQtBEX7lHMO7YRwg== + "@emotion/unitless@^0.8.1": version "0.8.1" resolved "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.8.1.tgz#182b5a4704ef8ad91bde93f7a860a88fd92c79a3" @@ -155,11 +292,21 @@ resolved "https://registry.npmjs.org/@emotion/use-insertion-effect-with-fallbacks/-/use-insertion-effect-with-fallbacks-1.0.1.tgz#08de79f54eb3406f9daaf77c76e35313da963963" integrity sha512-jT/qyKZ9rzLErtrjGgdkMBn2OP8wl0G3sQlBb3YPryvKHsjvINUhVaPFfP+fpBcOkmrVOVEEHQFJ7nbj2TH2gw== +"@emotion/utils@0.11.3": + version "0.11.3" + resolved "https://registry.npmjs.org/@emotion/utils/-/utils-0.11.3.tgz#a759863867befa7e583400d322652a3f44820924" + integrity sha512-0o4l6pZC+hI88+bzuaX/6BgOvQVhbt2PfmxauVaYOGgbsAw14wdKyvMCZXnsnsHys94iadcF+RG/wZyx6+ZZBw== + "@emotion/utils@^1.2.1": version "1.2.1" resolved "https://registry.npmjs.org/@emotion/utils/-/utils-1.2.1.tgz#bbab58465738d31ae4cb3dbb6fc00a5991f755e4" integrity sha512-Y2tGf3I+XVnajdItskUCn6LX+VUDmP6lTL4fcqsXAv43dnlbZiuW4MWQW38rW/BVWSE7Q/7+XQocmpnRYILUmg== +"@emotion/weak-memoize@0.2.5": + version "0.2.5" + resolved "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.2.5.tgz#8eed982e2ee6f7f4e44c253e12962980791efd46" + integrity sha512-6U71C2Wp7r5XtFtQzYrW5iKFT67OixrSxjI4MptCHzdSVlgabczzqLe0ZSgnub/5Kp4hSbpDB1tMytZY9pwxxA== + "@emotion/weak-memoize@^0.3.1": version "0.3.1" resolved "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.3.1.tgz#d0fce5d07b0620caa282b5131c297bb60f9d87e6" @@ -285,6 +432,38 @@ resolved "https://registry.npmjs.org/@fontsource/roboto/-/roboto-4.5.5.tgz" integrity sha512-Pe1p+gAO6K0aLxBXlLoJRHVx352tVc/v/7DOnvM3t+FYXb+KUga9aCD1NpnDfd0kKnWXqrZyAXguyyFWDDuphw== +"@jridgewell/gen-mapping@^0.3.5": + version "0.3.5" + resolved "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz#dcce6aff74bdf6dad1a95802b69b04a2fcb1fb36" + integrity sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg== + dependencies: + "@jridgewell/set-array" "^1.2.1" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping" "^0.3.24" + +"@jridgewell/resolve-uri@^3.1.0": + version "3.1.2" + resolved "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz#7a0ee601f60f99a20c7c7c5ff0c80388c1189bd6" + integrity sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw== + +"@jridgewell/set-array@^1.2.1": + version "1.2.1" + resolved "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz#558fb6472ed16a4c850b889530e6b36438c49280" + integrity sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A== + +"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.14": + version "1.5.0" + resolved "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz#3188bcb273a414b0d215fd22a58540b989b9409a" + integrity sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ== + +"@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25": + version "0.3.25" + resolved "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz#15f190e98895f3fc23276ee14bc76b675c2e50f0" + integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ== + dependencies: + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" + "@mui/base@5.0.0-beta.7": version "5.0.0-beta.7" resolved "https://registry.npmjs.org/@mui/base/-/base-5.0.0-beta.7.tgz#01cb99ac098af0ba989c7abc1474e3291c29414f" @@ -962,6 +1141,31 @@ array-flat-polyfill@^1.0.1: resolved "https://registry.npmjs.org/array-flat-polyfill/-/array-flat-polyfill-1.0.1.tgz" integrity sha512-hfJmKupmQN0lwi0xG6FQ5U8Rd97RnIERplymOv/qpq8AoNKPPAnxJadjFA23FNWm88wykh9HmpLJUUwUtNU/iw== +babel-plugin-emotion@^10.0.27: + version "10.2.2" + resolved "https://registry.npmjs.org/babel-plugin-emotion/-/babel-plugin-emotion-10.2.2.tgz#a1fe3503cff80abfd0bdda14abd2e8e57a79d17d" + integrity sha512-SMSkGoqTbTyUTDeuVuPIWifPdUGkTk1Kf9BWRiXIOIcuyMfsdp2EjeiiFvOzX8NOBvEh/ypKYvUh2rkgAJMCLA== + dependencies: + "@babel/helper-module-imports" "^7.0.0" + "@emotion/hash" "0.8.0" + "@emotion/memoize" "0.7.4" + "@emotion/serialize" "^0.11.16" + babel-plugin-macros "^2.0.0" + babel-plugin-syntax-jsx "^6.18.0" + convert-source-map "^1.5.0" + escape-string-regexp "^1.0.5" + find-root "^1.1.0" + source-map "^0.5.7" + +babel-plugin-macros@^2.0.0: + version "2.8.0" + resolved "https://registry.npmjs.org/babel-plugin-macros/-/babel-plugin-macros-2.8.0.tgz#0f958a7cc6556b1e65344465d99111a1e5e10138" + integrity sha512-SEP5kJpfGYqYKpBrj5XU3ahw5p5GOHJ0U5ssOSQ/WBVdwkD2Dzlce95exQTs3jOVWPPKLBN2rlEWkCK7dSmLvg== + dependencies: + "@babel/runtime" "^7.7.2" + cosmiconfig "^6.0.0" + resolve "^1.12.0" + babel-plugin-macros@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/babel-plugin-macros/-/babel-plugin-macros-3.1.0.tgz" @@ -971,6 +1175,11 @@ babel-plugin-macros@^3.1.0: cosmiconfig "^7.0.0" resolve "^1.19.0" +babel-plugin-syntax-jsx@^6.18.0: + version "6.18.0" + resolved "https://registry.npmjs.org/babel-plugin-syntax-jsx/-/babel-plugin-syntax-jsx-6.18.0.tgz#0af32a9a6e13ca7a3fd5069e62d7b0f58d0d8946" + integrity sha512-qrPaCSo9c8RHNRHIotaufGbuOBN8rtdC4QrrFFc43vyWCCz7Kl7GL1PGaXtMGQZUXrkCjNEgxDfmAuAabr/rlw== + binary-extensions@^2.0.0: version "2.3.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522" @@ -988,7 +1197,7 @@ callsites@^3.0.0: resolved "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== -chalk@^2.0.0: +chalk@^2.0.0, chalk@^2.4.2: version "2.4.2" resolved "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== @@ -1012,6 +1221,11 @@ chalk@^2.0.0: optionalDependencies: fsevents "~2.3.2" +classnames@^2.2.6: + version "2.5.1" + resolved "https://registry.npmjs.org/classnames/-/classnames-2.5.1.tgz#ba774c614be0f016da105c858e7159eae8e7687b" + integrity sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow== + classnames@^2.3.1: version "2.3.2" resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.3.2.tgz#351d813bf0137fcc6a76a16b88208d2560a0d924" @@ -1080,6 +1294,17 @@ convert-source-map@^1.5.0: resolved "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz#7faae62353fb4213366d0ca98358d22e8368b05f" integrity sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A== +cosmiconfig@^6.0.0: + version "6.0.0" + resolved "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-6.0.0.tgz#da4fee853c52f6b1e6935f41c1a2fc50bd4a9982" + integrity sha512-xb3ZL6+L8b9JLLCx3ZdoZy4+2ECphCMo2PwqgP1tlfVq6M6YReyzBJtvWWtbDSpNr9hn96pkCiZqUcFEc+54Qg== + dependencies: + "@types/parse-json" "^4.0.0" + import-fresh "^3.1.0" + parse-json "^5.0.0" + path-type "^4.0.0" + yaml "^1.7.2" + cosmiconfig@^7.0.0: version "7.0.1" resolved "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.0.1.tgz" @@ -1091,6 +1316,21 @@ cosmiconfig@^7.0.0: path-type "^4.0.0" yaml "^1.10.0" +create-emotion@^10.0.14, create-emotion@^10.0.27: + version "10.0.27" + resolved "https://registry.npmjs.org/create-emotion/-/create-emotion-10.0.27.tgz#cb4fa2db750f6ca6f9a001a33fbf1f6c46789503" + integrity sha512-fIK73w82HPPn/RsAij7+Zt8eCE8SptcJ3WoRMfxMtjteYxud8GDTKKld7MYwAX2TVhrw29uR1N/bVGxeStHILg== + dependencies: + "@emotion/cache" "^10.0.27" + "@emotion/serialize" "^0.11.15" + "@emotion/sheet" "0.9.4" + "@emotion/utils" "0.11.3" + +csstype@^2.5.7: + version "2.6.21" + resolved "https://registry.npmjs.org/csstype/-/csstype-2.6.21.tgz#2efb85b7cc55c80017c66a5ad7cbd931fda3a90e" + integrity sha512-Z1PhmomIfypOpoMjRQB70jfvy/wxT50qW08YXO5lMIJkrdq4yOTR+AW7FqutScmB9NkLwxo+jU+kZLbofZZq/w== + csstype@^3.0.2, csstype@^3.1.2: version "3.1.2" resolved "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz#1d4bf9d572f11c14031f0436e1c10bc1f571f50b" @@ -1395,6 +1635,13 @@ d3@^7.3.0: d3-transition "3" d3-zoom "3" +debug@^4.3.1: + version "4.3.7" + resolved "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz#87945b4151a011d76d95a198d7111c865c360a52" + integrity sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ== + dependencies: + ms "^2.1.3" + delaunator@5: version "5.0.0" resolved "https://registry.npmjs.org/delaunator/-/delaunator-5.0.0.tgz" @@ -1402,6 +1649,11 @@ delaunator@5: dependencies: robust-predicates "^3.0.0" +diff@^4.0.1: + version "4.0.2" + resolved "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" + integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== + dnd-core@^16.0.1: version "16.0.1" resolved "https://registry.npmjs.org/dnd-core/-/dnd-core-16.0.1.tgz" @@ -1424,6 +1676,14 @@ emoji-regex@^8.0.0: resolved "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz" integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== +emotion@^10.0.14: + version "10.0.27" + resolved "https://registry.npmjs.org/emotion/-/emotion-10.0.27.tgz#f9ca5df98630980a23c819a56262560562e5d75e" + integrity sha512-2xdDzdWWzue8R8lu4G76uWX5WhyQuzATon9LmNeCy/2BHVC6dsEpfhN1a0qhELgtDVdjyEA6J8Y/VlI5ZnaH0g== + dependencies: + babel-plugin-emotion "^10.0.27" + create-emotion "^10.0.27" + error-ex@^1.3.1: version "1.3.2" resolved "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" @@ -1512,6 +1772,11 @@ function-bind@^1.1.1: resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== +function-bind@^1.1.2: + version "1.1.2" + resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" + integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== + get-caller-file@^2.0.5: version "2.0.5" resolved "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz" @@ -1524,6 +1789,11 @@ glob-parent@~5.1.2: dependencies: is-glob "^4.0.1" +globals@^11.1.0: + version "11.12.0" + resolved "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" + integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== + has-flag@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" @@ -1536,6 +1806,13 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" +hasown@^2.0.2: + version "2.0.2" + resolved "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" + integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== + dependencies: + function-bind "^1.1.2" + hoist-non-react-statics@^3.3.0, hoist-non-react-statics@^3.3.1, hoist-non-react-statics@^3.3.2: version "3.3.2" resolved "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz#ece0acaf71d62c2969c2ec59feff42a4b1a85b45" @@ -1565,7 +1842,7 @@ immutable@^4.0.0: resolved "https://registry.yarnpkg.com/immutable/-/immutable-4.3.6.tgz#6a05f7858213238e587fb83586ffa3b4b27f0447" integrity sha512-Ju0+lEMyzMVZarkTn/gqRpdqd5dOPaz1mCZ0SH3JV6iFw81PldE/PEB1hWVEA288HPt4WXW8O7AWxB10M+03QQ== -import-fresh@^3.2.1: +import-fresh@^3.1.0, import-fresh@^3.2.1: version "3.3.0" resolved "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz" integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== @@ -1590,6 +1867,13 @@ is-binary-path@~2.1.0: dependencies: binary-extensions "^2.0.0" +is-core-module@^2.13.0: + version "2.15.1" + resolved "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.1.tgz#a7363a25bee942fefab0de13bf6aa372c82dcc37" + integrity sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ== + dependencies: + hasown "^2.0.2" + is-core-module@^2.8.1: version "2.12.1" resolved "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.1.tgz#0c0b6885b6f80011c71541ce15c8d66cf5a4f9fd" @@ -1624,6 +1908,11 @@ is-number@^7.0.0: resolved "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== +jsesc@^2.5.1: + version "2.5.2" + resolved "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" + integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== + json-parse-even-better-errors@^2.3.0: version "2.3.1" resolved "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d" @@ -1670,6 +1959,16 @@ markdown-to-jsx@^7.1.8: resolved "https://registry.yarnpkg.com/markdown-to-jsx/-/markdown-to-jsx-7.1.8.tgz#49c3bb3c122aa714324034142c8829b93c889338" integrity sha512-rRSa1aFmFnpDRFAhv5vIkWM4nPaoB9vnzIjuIKa1wGupfn2hdCNeaQHKpu4/muoc8n8J7yowjTP2oncA4/Rbgg== +memoize-one@^5.0.4: + version "5.2.1" + resolved "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz#8337aa3c4335581839ec01c3d594090cebe8f00e" + integrity sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q== + +ms@^2.1.3: + version "2.1.3" + resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + mui-markdown@^1.1.13: version "1.1.13" resolved "https://registry.npmjs.org/mui-markdown/-/mui-markdown-1.1.13.tgz#749b3c77379924985c016c907dca1e5423bf9590" @@ -1726,6 +2025,11 @@ path-type@^4.0.0: resolved "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== +picocolors@^1.0.0: + version "1.1.0" + resolved "https://registry.npmjs.org/picocolors/-/picocolors-1.1.0.tgz#5358b76a78cde483ba5cef6a9dc9671440b27d59" + integrity sha512-TQ92mBOW0l3LeMeyLV6mzy/kWr8lkd/hp3mTg7wYK7zJhuBStmGMBG0BdeDZS/dZx1IukaX6Bk11zcln25o1Aw== + picocolors@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1" @@ -1789,6 +2093,18 @@ react-animate-on-change@^2.2.0: resolved "https://registry.npmjs.org/react-animate-on-change/-/react-animate-on-change-2.2.0.tgz" integrity sha512-cM0YHbsxIh8fshX/U24+pk4nDG7Ike9NsEy21reqJPqVt6xRA+6oYkaQHEggINKjYEMbztwK40Ro0/EHZ5naVQ== +react-diff-viewer@^3.1.1: + version "3.1.1" + resolved "https://registry.npmjs.org/react-diff-viewer/-/react-diff-viewer-3.1.1.tgz#21ac9c891193d05a3734bfd6bd54b107ee6d46cc" + integrity sha512-rmvwNdcClp6ZWdS11m1m01UnBA4OwYaLG/li0dB781e/bQEzsGyj+qewVd6W5ztBwseQ72pO7nwaCcq5jnlzcw== + dependencies: + classnames "^2.2.6" + create-emotion "^10.0.14" + diff "^4.0.1" + emotion "^10.0.14" + memoize-one "^5.0.4" + prop-types "^15.6.2" + react-dnd-html5-backend@^16.0.1: version "16.0.1" resolved "https://registry.npmjs.org/react-dnd-html5-backend/-/react-dnd-html5-backend-16.0.1.tgz" @@ -1944,6 +2260,11 @@ regenerator-runtime@^0.13.11, regenerator-runtime@^0.13.4: resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz#f6dca3e7ceec20590d07ada785636a90cdca17f9" integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg== +regenerator-runtime@^0.14.0: + version "0.14.1" + resolved "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz#356ade10263f685dda125100cd862c1db895327f" + integrity sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw== + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz" @@ -1959,6 +2280,15 @@ resolve-from@^4.0.0: resolved "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== +resolve@^1.12.0: + version "1.22.8" + resolved "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" + integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== + dependencies: + is-core-module "^2.13.0" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" + resolve@^1.19.0: version "1.22.0" resolved "https://registry.npmjs.org/resolve/-/resolve-1.22.0.tgz" @@ -2630,7 +2960,7 @@ y18n@^5.0.5: resolved "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== -yaml@^1.10.0: +yaml@^1.10.0, yaml@^1.7.2: version "1.10.2" resolved "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz" integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== From ff6b585fef3dc12932204be2124343d66a798353 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Fri, 4 Oct 2024 12:38:58 -0700 Subject: [PATCH 03/39] wip --- src/views/TableSelectionView.tsx | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/views/TableSelectionView.tsx b/src/views/TableSelectionView.tsx index fab76c8..7f7a178 100644 --- a/src/views/TableSelectionView.tsx +++ b/src/views/TableSelectionView.tsx @@ -413,6 +413,8 @@ export const TableCopyDialogV2: React.FC = ({ buttonElemen const [tableName, setTableName] = useState(""); const [tableContent, setTableContent] = useState(""); + const [rawImgContent, setRawImgContent] = useState(""); + const [cleaningInProgress, setCleaningInProgress] = useState(false); const [cleanTableContent, setCleanTableContent] = useState<{content: string, reason: string, mode: string} | undefined>(undefined); @@ -593,7 +595,7 @@ export const TableCopyDialogV2: React.FC = ({ buttonElemen {cleaningInProgress ? : ""} - { viewTable ? + {viewTable ? <> {/* = ({ buttonElemen { setTableContent(event.target.value); }} + onChange={(event) => { + setTableContent(event.target.value); + }} + onPasteCapture={(e) => { + console.log(e.clipboardData.files); + if (e.clipboardData.files.length > 0) { + var file = e.clipboardData.files[0], + read = new FileReader(); + + read.readAsDataURL(file); + + read.onloadend = function(){ + let res = read.result; + console.log(res); + if (res) { setRawImgContent(res as string); } + } + } + }} autoComplete='off' label="content (csv, tsv, or json format)" variant="outlined" multiline minRows={15} /> } + + Red dot { cleanTableContent != undefined ? From ea0c56282a74e7c1043b289c5d864f7436ca9331 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Mon, 7 Oct 2024 12:17:18 -0700 Subject: [PATCH 04/39] some fixes --- app.py | 2 +- server/agents/agent_data_clean.py | 45 ++++++++++++++++++++++----- server/agents/agent_data_load.py | 3 ++ src/views/TableSelectionView.tsx | 51 ++++++++++++++++++++++--------- 4 files changed, 78 insertions(+), 23 deletions(-) diff --git a/app.py b/app.py index 4707c6d..7a18b1d 100644 --- a/app.py +++ b/app.py @@ -287,7 +287,7 @@ def clean_data_request(): agent = DataCleanAgent(client=client, model=model) - candidates = agent.run(content["raw_data"]) + candidates = agent.run(content['content_type'], content["raw_data"]) candidates = [c for c in candidates if c['status'] == 'ok'] diff --git a/server/agents/agent_data_clean.py b/server/agents/agent_data_clean.py index 34ab4f4..b61b4af 100644 --- a/server/agents/agent_data_clean.py +++ b/server/agents/agent_data_clean.py @@ -56,6 +56,8 @@ - If the user doesn't indicate how many rows to be generated, plan in generating a dataset with 10-20 rows depending on the content. ''' + + EXAMPLE = ''' [RAW DATA] @@ -101,16 +103,45 @@ def __init__(self, client, model): self.model = model self.client = client - def run(self, raw_data): + def run(self, content_type, raw_data): """derive a new concept based on the raw input data """ - user_query = f"[DATA]\n\n{raw_data}\n\n[OUTPUT]\n" - - logger.info(user_query) - - messages = [{"role":"system", "content": SYSTEM_PROMPT}, - {"role":"user","content": user_query}] + if content_type == "text": + user_prompt = { + "role": "user", + "content": [{ + 'type': 'text', + 'text': f"[DATA]\n\n{raw_data}\n\n[OUTPUT]\n" + }] + } + elif content_type == "image": + user_prompt = { + 'role': 'user', + 'content': [ { + 'type': 'text', + 'text': '''[RAW_DATA]\n\n'''}, + { + 'type': 'image_url', + 'image_url': { + "url": raw_data, + "detail": "high" + } + }, + { + 'type': 'text', + 'text': '''[OUTPUT]\n\n''' + }, + ] + } + + logger.info(user_prompt) + + system_message = { + 'role': 'system', + 'content': [ {'type': 'text', 'text': SYSTEM_PROMPT}]} + + messages = [system_message, user_prompt] ###### the part that calls open_ai response = self.client.chat.completions.create( diff --git a/server/agents/agent_data_load.py b/server/agents/agent_data_load.py index 6470b05..4f0ec5f 100644 --- a/server/agents/agent_data_load.py +++ b/server/agents/agent_data_load.py @@ -18,6 +18,9 @@ Furthermore, if the field is string type and is ordinal (especially for english month name, week name, range), provide the natural sort order of the fields here. Otherwise, put sort_order as null (for example, Name should not be sorted). +Special cases: +* sometimes, column name is year like "2020", "2021" but its content is not actually year (e.g., sales), in these cases, the semantic type of the column would not be Year! + Create a json object function based off the [DATA] provided. output should be in the format of: diff --git a/src/views/TableSelectionView.tsx b/src/views/TableSelectionView.tsx index 7f7a178..a3cbc33 100644 --- a/src/views/TableSelectionView.tsx +++ b/src/views/TableSelectionView.tsx @@ -8,7 +8,8 @@ import Tab from '@mui/material/Tab'; import Typography from '@mui/material/Typography'; import Box from '@mui/material/Box'; import { alpha, Button, Collapse, Dialog, DialogActions, DialogContent, DialogTitle, Divider, - IconButton, Input, CircularProgress, LinearProgress, Paper, TextField, useTheme } from '@mui/material'; + IconButton, Input, CircularProgress, LinearProgress, Paper, TextField, useTheme, + Card} from '@mui/material'; import { CustomReactTable } from './ReactTable'; import { DictTable } from "../components/ComponentType"; @@ -411,10 +412,9 @@ export const TableCopyDialogV2: React.FC = ({ buttonElemen const [dialogOpen, setDialogOpen] = useState(false); const [tableName, setTableName] = useState(""); + const [tableContent, setTableContent] = useState(""); - - const [rawImgContent, setRawImgContent] = useState(""); - + const [tableContentType, setTableContentType] = useState<'text' | 'image'>('text'); const [cleaningInProgress, setCleaningInProgress] = useState(false); const [cleanTableContent, setCleanTableContent] = useState<{content: string, reason: string, mode: string} | undefined>(undefined); @@ -457,6 +457,7 @@ export const TableCopyDialogV2: React.FC = ({ buttonElemen .then(content => { setTableName(tableName); setTableContent(content); + setTableContentType("text"); }) } @@ -470,6 +471,7 @@ export const TableCopyDialogV2: React.FC = ({ buttonElemen headers: { 'Content-Type': 'application/json', }, body: JSON.stringify({ token: token, + content_type: tableContentType, raw_data: tableContent, model: activeModel }), @@ -612,10 +614,11 @@ export const TableCopyDialogV2: React.FC = ({ buttonElemen rowsPerPageNum={-1} compact={false} columnDefs={viewTable.names.map(name => { return { id: name, label: name, minWidth: 60, align: undefined, format: (v: any) => v} - })} /> + })} + /> {/* {cleanTableContent.reason} */} - : + : ( tableContentType == "text" ? = ({ buttonElemen onPasteCapture={(e) => { console.log(e.clipboardData.files); if (e.clipboardData.files.length > 0) { - var file = e.clipboardData.files[0], - read = new FileReader(); + let file = e.clipboardData.files[0]; + let read = new FileReader(); read.readAsDataURL(file); read.onloadend = function(){ let res = read.result; console.log(res); - if (res) { setRawImgContent(res as string); } + if (res) { + setTableContent(res as string); + setTableContentType("image"); + } } } }} autoComplete='off' label="content (csv, tsv, or json format)" variant="outlined" multiline minRows={15} /> + : + + { + setTableContent(""); + setTableContentType("text"); + }} + > + + + the image is corrupted, please try again. + ) } - - - Red dot + { cleanTableContent != undefined ? @@ -652,19 +669,23 @@ export const TableCopyDialogV2: React.FC = ({ buttonElemen Revert : } {/* */} -