diff --git a/.dvc/config b/.dvc/config index 965664e..a364124 100644 --- a/.dvc/config +++ b/.dvc/config @@ -5,4 +5,4 @@ [cache] dir = cache ['remote "default-s3-remote"'] - url = s3://lpm-research/default + url = s3://lpm-research/e-stat_japan diff --git a/data/.dvc/.gitignore b/data/.dvc/.gitignore new file mode 100644 index 0000000..528f30c --- /dev/null +++ b/data/.dvc/.gitignore @@ -0,0 +1,3 @@ +/config.local +/tmp +/cache diff --git a/data/data.csv.dvc b/data/data.csv.dvc new file mode 100644 index 0000000..9ab7a25 --- /dev/null +++ b/data/data.csv.dvc @@ -0,0 +1,5 @@ +outs: +- md5: 6d50ebcd33da129d90f40d7bdf39d7cc + size: 91972 + hash: md5 + path: data.csv diff --git a/data/fidelity/.gitignore b/data/fidelity/.gitignore new file mode 100644 index 0000000..c0e4f1d --- /dev/null +++ b/data/fidelity/.gitignore @@ -0,0 +1,3 @@ +/synthetic-data-gensql.csv +/synthetic-data-gan.csv +/synthetic-data-copula.csv diff --git a/data/synthetic-data-copula.csv b/data/synthetic-data-copula.csv new file mode 100644 index 0000000..1b834e0 --- /dev/null +++ b/data/synthetic-data-copula.csv @@ -0,0 +1,1001 @@ +Total_population__Both_sexes_,Taxable_income__million_yen_,Ordinal_Number_of_employed_persons,Number_of_households__Total_,Ordinal_Number_of_physicians,Number_of_dwellings_with_occupying_households,Ordinal_Total_population__65_and_over_,Number_of_lower_secondary_school_students,Number_of_elderly_nursing_facilities__Questionnaire_about_basic_matters_ +b,b,e,b,a,,d,b,c +c,c,f,c,a,c,d,c,b +b,d,f,a,a,d,e,a,b +c,d,e,c,a,,e,d,c +c,c,e,d,a,d,c,c,c +a,,e,a,a,,e,a,a +c,d,f,d,a,b,c,c,d +b,b,e,b,a,a,d,b,a +d,d,e,d,a,c,d,c,b +c,d,e,c,a,b,b,c,c +a,a,e,a,a,c,d,c,d +c,a,e,c,a,c,d,c,a +a,c,f,a,a,c,d,a,a +c,d,e,d,a,,d,d,d +a,b,e,c,a,c,e,a,a +d,d,e,c,a,,d,d,b +c,c,e,b,a,d,e,a,c +d,d,e,c,a,c,d,d,d +c,c,e,d,a,c,e,c,b +b,b,e,b,a,b,c,b,a +b,b,e,b,a,a,c,b,a +c,c,e,a,a,b,f,a,c +d,d,e,d,a,d,b,d,d +b,b,e,b,a,,c,b,c +a,c,f,a,a,d,d,a,a +c,c,e,c,a,d,d,d,b +d,d,e,d,a,,b,d,a +c,b,e,c,a,c,d,a,a +b,a,f,a,a,,e,b,a +a,b,f,b,a,d,d,a,a +c,d,e,d,a,c,e,d,b +d,c,e,c,a,a,c,a,d +b,a,e,b,a,a,c,a,a +b,a,e,a,a,,d,a,d +c,,e,a,a,,c,c,c +a,b,e,b,a,d,e,b,c +d,d,e,d,a,d,b,d,d +d,c,e,d,a,,e,d,d +c,a,e,c,a,,d,a,a +b,b,f,b,a,d,d,b,b +c,d,e,d,a,,d,d,b +d,c,e,c,a,b,c,b,a +a,a,f,b,a,,d,a,a +a,c,f,b,a,,c,a,a +b,b,f,b,a,,c,b,b +c,d,f,d,a,,e,c,d +c,,e,c,a,d,c,d,a +c,d,f,c,a,,c,c,d +b,b,e,b,a,a,d,b,a +b,a,e,b,a,c,d,a,a +c,,f,a,a,,c,a,d +c,d,e,a,a,a,d,c,a +c,c,e,d,a,c,c,c,b +c,c,f,c,a,c,c,c,c +b,a,f,b,a,b,d,a,a +b,b,e,b,a,a,c,b,a +d,d,e,d,a,,c,d,d +b,b,e,a,a,a,d,a,a +d,c,e,c,a,b,d,d,d +a,c,e,a,a,,c,a,a +b,b,e,b,a,b,d,b,a +d,c,e,d,a,,c,d,b +c,a,e,c,a,,f,c,d +c,a,e,c,a,a,c,a,c +c,d,f,c,a,b,d,d,b +c,d,e,c,a,,e,d,d +a,b,e,c,a,c,e,b,c +c,,e,c,a,b,e,c,a +c,d,f,a,a,a,d,c,b +b,b,f,b,a,c,d,b,a +a,b,f,a,a,d,b,a,c +c,d,e,d,a,c,c,d,b +b,d,f,a,a,,d,c,c +b,c,e,a,a,b,c,a,a +b,b,e,b,a,d,c,b,a +c,a,e,c,a,b,f,c,c +c,,e,a,a,,c,a,d +d,a,f,d,a,d,d,c,b +c,d,e,d,a,a,d,c,b +b,b,e,b,a,,d,b,b +a,,f,d,a,a,c,a,d +b,,e,b,a,,f,b,c +b,b,d,b,a,a,d,b,a +c,c,e,c,a,c,c,c,a +a,b,f,a,a,a,f,b,a +a,b,e,c,a,,e,a,c +a,a,e,a,a,a,d,b,b +b,b,d,b,a,a,f,b,a +b,b,f,a,a,d,d,a,a +c,,e,d,a,c,d,a,c +b,a,e,b,a,a,d,b,a +c,a,f,a,a,d,d,a,a +c,d,e,c,a,b,e,d,b +b,b,f,b,a,,d,b,a +b,a,g,c,a,,e,a,a +b,b,f,b,a,a,c,b,a +c,c,e,c,a,b,e,d,b +c,c,e,a,a,b,e,c,a +b,b,e,b,a,a,d,b,a +a,a,e,b,a,c,d,a,b +b,b,e,b,a,a,d,b,c +a,c,e,c,a,,d,a,a +c,d,e,d,a,b,b,d,a +a,c,d,a,a,d,d,a,b +c,,f,c,a,c,d,c,a +c,d,f,d,a,c,c,c,b +d,d,e,d,a,b,c,d,b +c,c,f,c,a,,c,c,b +b,a,e,b,a,,c,c,c +d,c,f,d,a,,c,d,b +c,c,f,a,a,d,c,b,b +a,a,e,a,a,d,d,b,c +b,b,e,b,a,a,f,b,c +c,,e,a,a,a,d,a,d +d,d,f,d,a,b,d,d,d +c,a,e,b,a,d,d,a,c +d,a,e,c,a,b,e,d,a +b,b,e,b,a,,c,a,a +b,b,f,b,a,d,c,b,c +b,b,f,b,a,a,d,b,a +a,b,e,a,a,,c,b,a +a,b,e,a,a,a,d,b,b +a,a,e,a,a,c,d,b,a +a,c,f,b,a,,f,a,a +a,a,e,a,a,,c,c,b +d,d,e,d,a,c,d,d,d +a,c,e,a,a,,d,c,b +c,c,e,d,a,c,e,a,d +c,c,e,c,a,a,c,d,d +d,d,f,d,a,c,d,d,c +b,b,f,a,a,a,c,a,a +b,a,e,a,a,,d,a,b +b,a,e,a,a,,d,c,b +a,d,e,a,a,d,e,c,b +a,b,e,b,a,a,e,b,b +a,b,f,c,a,b,c,b,a +d,,e,d,a,d,c,c,d +b,a,e,b,a,a,e,b,a +b,b,f,b,a,d,d,b,c +a,,f,a,a,,d,a,a +a,b,e,a,a,,e,a,a +b,b,e,a,a,b,d,b,a +a,a,f,b,a,,d,b,a +a,b,e,a,a,,d,c,b +a,,e,a,a,,d,c,b +b,a,f,b,a,,d,b,a +c,d,e,c,a,d,d,c,a +d,a,f,d,a,c,e,d,d +c,b,f,a,a,a,e,c,c +d,c,e,d,a,c,e,d,a +a,b,e,a,a,,c,a,c +a,d,g,a,a,d,d,a,c +d,d,e,d,a,c,c,d,a +d,d,e,d,a,c,f,d,b +b,b,f,b,a,a,e,b,b +c,a,f,a,a,c,b,a,a +d,c,e,c,a,c,c,a,a +b,b,e,a,a,a,d,b,a +b,a,f,a,a,,c,a,d +a,d,f,b,a,,d,c,a +b,a,f,b,a,,e,b,a +d,c,e,c,a,b,c,d,b +d,c,f,d,a,,e,c,a +c,b,d,c,a,a,c,a,b +d,c,e,d,a,b,d,d,d +b,a,f,b,a,b,d,a,a +a,a,e,c,a,,d,c,a +d,c,e,d,a,b,e,c,c +d,c,e,d,a,,f,d,c +b,a,e,a,a,a,f,b,a +b,a,f,a,a,,c,b,b +b,b,f,b,a,b,d,b,a +d,d,e,d,a,d,e,d,b +a,,e,c,a,b,b,c,a +a,c,e,a,a,a,d,a,d +a,c,e,c,a,a,b,a,d +d,c,f,d,a,,c,d,c +d,d,e,d,a,c,d,d,b +a,b,e,a,a,,f,b,d +c,c,e,d,a,b,e,c,d +c,,f,a,a,,d,b,b +a,b,f,c,a,,d,b,a +b,b,e,a,a,,d,a,c +d,,e,d,a,,e,c,c +c,d,f,c,a,c,e,c,d +a,a,e,b,a,a,d,a,a +d,c,e,d,a,b,e,d,b +b,b,e,b,a,a,e,b,a +b,b,e,b,a,a,e,b,a +a,c,f,b,a,,e,b,a +d,d,e,c,a,c,d,d,b +c,a,e,a,a,d,e,a,d +d,d,e,d,a,b,b,d,d +b,d,e,a,a,c,e,c,a +d,d,e,d,a,b,e,c,a +b,b,f,b,a,d,d,b,a +c,d,e,c,a,d,c,a,d +b,c,f,b,a,,c,b,a +a,a,e,a,a,,c,c,b +d,,e,a,a,,e,d,a +a,d,e,b,a,a,f,a,a +b,a,e,b,a,d,c,a,c +c,c,e,c,a,,e,c,a +b,c,e,a,a,b,c,b,b +d,,e,d,a,d,e,c,c +b,b,e,b,a,c,c,a,a +b,a,e,a,a,,c,a,a +b,b,f,b,a,d,c,b,a +b,b,d,b,a,d,c,a,a +d,d,f,d,a,c,d,d,a +a,,e,a,a,b,c,a,d +a,,e,c,a,,d,c,d +b,b,f,b,a,,e,b,c +c,a,e,d,a,b,c,c,b +b,a,e,a,a,c,e,b,a +d,d,f,c,a,c,e,d,d +a,,e,d,a,b,d,c,c +d,c,e,d,a,,c,d,c +a,b,f,b,a,,c,b,a +b,b,f,b,a,a,e,a,a +a,b,e,c,a,c,c,c,a +a,d,e,a,a,d,c,c,b +b,a,f,b,a,b,d,a,a +c,a,e,d,a,d,d,c,d +a,a,e,a,a,b,c,c,a +c,,e,c,a,c,b,a,b +b,b,e,b,a,d,d,b,a +c,a,e,c,a,c,d,c,c +c,d,e,a,a,c,c,d,c +d,d,e,d,a,c,c,d,b +b,b,f,b,a,d,f,b,b +b,b,e,a,a,,c,b,a +c,d,e,d,a,,c,d,c +d,b,f,d,a,d,c,d,a +b,a,e,b,a,,c,b,a +b,a,e,b,a,a,d,b,a +c,d,e,c,a,b,e,d,a +b,b,h,b,a,b,d,b,a +b,b,e,b,a,d,c,b,a +a,b,e,a,a,d,c,a,d +a,b,f,c,a,b,d,a,b +a,b,e,b,a,,d,a,a +a,,f,a,a,a,d,b,a +d,c,e,d,a,,c,d,b +d,,e,c,a,b,e,c,d +b,,f,b,a,b,c,c,a +c,d,f,a,a,d,d,a,d +d,c,e,d,a,,c,d,d +b,b,f,b,a,a,e,b,a +a,c,e,c,a,c,e,c,d +a,,f,c,a,b,f,a,a +c,c,e,a,a,b,d,c,b +b,b,e,b,a,a,d,b,d +b,d,f,a,a,c,d,a,a +d,d,e,d,a,,c,c,d +a,a,e,c,a,a,d,b,d +c,a,e,a,a,,f,a,b +c,c,e,d,a,a,d,d,b +a,a,e,c,a,,c,c,a +b,c,f,b,a,c,d,b,a +b,b,f,b,a,,c,a,a +a,a,e,b,a,,c,b,c +b,a,e,a,a,,d,a,b +d,d,e,d,a,b,c,d,a +a,a,e,b,a,d,c,a,c +b,,f,a,a,d,d,b,b +a,a,e,d,a,d,f,c,c +b,b,f,b,a,,d,c,c +d,d,e,c,a,a,d,d,b +a,,e,a,a,c,c,a,a +b,a,e,b,a,d,c,b,a +a,d,e,a,a,d,c,c,b +c,,f,c,a,a,c,a,b +b,c,e,b,a,c,c,a,a +c,c,e,a,a,d,c,c,c +a,a,e,a,a,b,d,c,c +d,d,e,d,a,c,d,d,d +c,c,e,c,a,c,d,c,a +a,b,e,a,a,c,d,c,b +b,a,f,b,a,d,c,b,a +a,,f,a,a,a,c,a,b +c,c,e,c,a,d,c,d,a +d,c,e,d,a,b,c,d,b +b,b,e,b,a,d,e,a,c +a,,e,b,a,d,d,b,b +a,c,f,a,a,,d,a,d +d,a,e,c,a,a,c,d,d +b,,e,a,a,c,c,a,b +b,d,e,b,a,d,d,c,a +a,,f,d,a,c,d,c,b +a,b,e,a,a,,f,a,c +a,c,e,b,a,c,d,a,a +b,,f,d,a,b,e,a,a +c,d,e,d,a,d,c,d,d +d,a,f,c,a,,d,c,b +a,c,e,c,a,a,c,a,d +a,a,f,a,a,,d,a,d +d,a,f,c,a,b,d,c,a +a,a,e,a,a,b,e,c,a +b,a,e,a,a,,d,a,a +a,c,f,a,a,a,d,b,b +c,c,e,d,a,,c,d,d +c,b,e,c,a,,f,a,c +c,a,f,a,a,b,b,c,b +d,d,f,d,a,b,e,d,c +c,a,f,d,a,d,c,a,a +a,b,e,a,a,a,c,a,c +b,b,f,b,a,,d,b,a +a,c,e,c,a,b,d,a,b +d,d,e,d,a,b,c,d,b +b,b,f,b,a,d,d,b,a +c,b,e,b,a,,d,a,c +b,d,f,b,a,a,d,b,a +d,a,e,d,a,b,d,a,b +c,a,f,a,a,,c,b,a +b,b,e,b,a,d,c,b,a +a,b,d,b,a,d,d,b,a +a,a,e,a,a,d,d,a,b +d,d,f,d,a,c,d,d,b +c,d,e,d,a,b,c,a,a +d,d,e,d,a,c,e,d,b +b,b,e,b,a,a,c,b,a +a,a,e,d,a,b,e,d,a +c,,f,a,a,d,c,c,b +a,c,f,a,a,,d,c,d +c,,e,d,a,d,d,d,a +d,c,e,a,a,,d,d,d +a,d,e,c,a,,d,c,b +b,a,e,b,a,,e,a,a +d,d,f,d,a,,c,d,d +b,b,h,b,a,d,d,b,a +a,,d,c,a,,d,a,a +c,c,f,d,a,a,d,c,a +c,a,f,a,a,,d,c,d +a,d,e,c,a,b,d,d,b +c,d,e,c,a,a,d,d,b +c,d,e,a,a,b,c,d,d +b,b,f,b,a,a,d,b,b +a,c,e,a,a,,d,c,d +a,b,e,c,a,b,f,a,a +b,a,d,b,a,b,d,b,c +a,d,e,c,a,,c,c,b +a,,e,b,a,d,c,d,d +c,c,f,c,a,c,c,a,c +d,d,e,d,a,,d,d,d +b,b,f,b,a,,e,b,b +c,c,e,d,a,,c,d,d +c,d,e,a,a,a,e,c,d +a,,d,c,a,d,c,b,b +c,,e,c,a,,c,a,b +b,b,e,b,a,b,b,b,a +a,b,e,b,a,c,c,b,a +c,c,f,c,a,c,c,d,a +b,c,e,b,a,a,c,b,d +d,c,f,a,a,c,c,d,b +d,d,e,d,a,a,c,d,d +b,a,f,a,a,b,c,c,b +c,c,f,c,a,c,f,a,d +c,a,e,c,a,,d,c,d +b,a,e,b,a,d,d,b,a +a,b,f,d,a,,b,a,c +c,c,e,c,a,,c,d,d +a,a,f,b,a,,c,b,c +c,c,e,c,a,,d,c,b +c,c,e,a,a,c,e,a,a +d,c,e,d,a,c,d,a,d +a,c,e,a,a,c,d,c,b +d,c,e,d,a,,d,d,d +d,a,e,c,a,c,c,c,c +d,c,e,a,a,,e,a,b +b,a,f,a,a,c,d,b,a +b,a,f,c,a,c,b,b,c +b,b,e,b,a,d,c,b,a +c,a,e,c,a,b,c,a,b +c,,e,c,a,,d,c,c +c,d,e,c,a,c,c,d,a +a,a,e,b,a,a,e,a,d +d,d,e,a,a,c,c,d,b +c,c,f,c,a,,f,c,a +b,b,e,b,a,a,e,b,c +d,d,e,d,a,c,c,d,c +d,c,e,c,a,c,d,d,a +d,d,e,d,a,,d,d,d +c,,e,c,a,c,c,c,d +b,d,e,b,a,c,d,c,c +c,a,f,c,a,,d,c,c +c,d,e,c,a,a,c,d,c +d,d,e,d,a,c,d,d,d +a,a,e,a,a,d,e,c,d +a,a,e,a,a,,e,c,d +b,b,d,b,a,a,d,b,a +a,a,e,c,a,d,d,a,d +d,b,e,c,a,a,c,a,d +b,b,e,b,a,,c,b,a +d,d,e,d,a,d,d,d,d +d,c,f,d,a,c,d,d,d +d,d,e,d,a,c,e,d,b +c,c,e,d,a,,e,d,c +d,d,f,d,a,d,c,d,b +d,,f,d,a,b,c,d,d +b,b,f,a,a,,e,b,a +c,d,e,c,a,c,c,c,c +d,a,e,c,a,d,c,c,d +a,c,e,a,a,b,c,a,d +c,a,f,a,a,,e,b,c +d,d,f,c,a,c,c,d,b +d,c,f,c,a,,d,c,b +b,a,e,b,a,a,c,b,c +b,b,e,a,a,,e,a,b +a,a,e,b,a,b,d,b,c +b,b,e,a,a,a,d,a,a +d,c,e,d,a,a,d,c,b +b,b,f,b,a,b,c,b,a +b,b,e,b,a,a,c,b,b +a,c,e,a,a,,e,c,c +b,b,e,b,a,,c,a,b +b,,f,b,a,a,d,b,a +b,b,d,b,a,a,d,b,a +c,a,e,a,a,,b,c,b +a,,e,a,a,a,c,c,b +d,d,f,d,a,,d,d,d +d,c,e,a,a,,e,d,c +d,d,e,d,a,c,d,d,d +a,,e,a,a,d,c,a,b +a,a,f,a,a,,c,b,a +d,d,e,d,a,c,f,d,b +a,b,e,a,a,a,e,b,d +b,c,e,b,a,b,e,a,b +d,d,e,d,a,c,e,d,d +d,c,f,d,a,a,c,d,d +d,d,f,c,a,c,b,c,c +c,a,e,c,a,b,c,c,b +c,d,f,c,a,,d,d,d +a,,d,c,a,,d,b,a +c,a,e,c,a,b,f,c,c +b,b,e,b,a,a,c,b,c +c,c,f,a,a,b,d,c,d +d,d,e,c,a,c,e,d,c +c,d,f,d,a,,d,c,d +d,d,e,d,a,b,e,c,b +b,b,e,b,a,c,d,a,a +b,,e,b,a,,d,b,d +a,c,e,b,a,a,d,a,d +c,b,e,a,a,a,d,a,d +d,d,e,d,a,b,d,d,d +b,a,f,b,a,b,d,b,a +d,d,e,d,a,,d,d,d +c,c,e,c,a,c,d,a,d +d,a,e,a,a,d,e,b,c +b,b,f,b,a,b,c,b,a +c,c,e,a,a,c,d,c,a +a,b,e,c,a,c,e,d,b +d,c,e,d,a,b,d,c,a +c,d,e,c,a,,d,d,a +c,c,e,c,a,a,c,d,d +c,c,e,a,a,a,d,d,c +b,a,f,b,a,a,c,a,b +c,c,e,a,a,,d,c,d +b,a,e,b,a,b,c,b,b +a,a,e,b,a,a,c,a,c +d,d,e,c,a,a,c,d,d +b,b,e,b,a,a,e,b,a +b,a,f,b,a,a,e,b,b +d,d,e,d,a,b,d,d,d +a,c,e,a,a,b,c,c,b +a,,d,c,a,d,d,a,b +d,d,f,d,a,c,d,d,d +b,a,e,b,a,a,e,b,a +b,d,e,a,a,a,d,a,b +c,,e,a,a,b,c,c,a +d,,e,c,a,a,c,d,c +c,a,e,c,a,a,e,a,d +a,b,f,c,a,a,e,a,b +b,a,f,a,a,d,e,b,a +c,a,e,d,a,,f,c,d +c,d,f,c,a,,c,c,a +b,c,e,a,a,b,c,a,c +d,d,e,d,a,d,c,d,b +a,b,f,b,a,,e,b,a +b,a,e,b,a,,d,c,b +b,b,e,b,a,a,c,b,a +d,d,e,c,a,,e,d,d +c,a,e,c,a,,c,d,c +d,d,e,d,a,a,d,d,d +d,d,f,d,a,,d,c,c +c,c,e,d,a,,d,d,c +a,d,e,d,a,c,e,c,c +c,a,e,d,a,,d,c,b +c,c,e,c,a,b,e,d,b +b,,f,b,a,b,e,b,a +a,b,f,a,a,,c,a,a +c,a,f,c,a,a,d,a,c +c,a,e,a,a,a,d,b,c +c,c,e,a,a,,d,a,b +b,a,e,a,a,a,c,b,d +a,b,e,a,a,d,f,b,a +a,a,e,b,a,,c,a,a +b,a,g,b,a,d,d,a,a +b,b,f,b,a,d,c,b,a +a,d,e,c,a,c,c,c,a +b,b,e,b,a,,d,b,a +d,d,e,c,a,d,b,d,d +b,b,e,b,a,a,c,a,a +c,b,e,c,a,a,d,a,b +c,d,f,d,a,c,c,d,a +c,c,f,b,a,,d,d,b +a,a,e,c,a,d,c,c,d +a,b,d,b,a,a,c,a,a +a,a,f,c,a,,c,a,c +d,d,f,d,a,,e,d,d +a,b,e,b,a,a,e,a,a +b,c,e,a,a,a,d,b,d +d,d,e,d,a,b,d,d,d +c,,e,c,a,a,d,d,d +b,c,e,c,a,c,e,a,a +c,c,f,d,a,,c,a,d +c,a,e,a,a,b,e,a,d +b,b,e,b,a,d,f,b,c +b,b,e,b,a,,c,b,c +a,b,e,c,a,a,c,b,d +b,b,f,a,a,a,e,b,c +c,d,f,a,a,b,c,a,a +c,b,e,c,a,,c,c,a +b,a,e,a,a,,c,b,a +c,a,e,c,a,,e,a,d +c,,e,a,a,d,e,a,c +a,a,f,a,a,c,e,a,a +b,b,g,b,a,a,f,b,a +b,b,f,a,a,a,c,b,a +a,a,f,a,a,,d,b,a +d,d,f,d,a,,d,d,d +d,d,e,d,a,c,d,d,d +a,a,e,c,a,,d,a,d +c,a,e,c,a,c,e,c,c +a,a,e,a,a,a,d,b,b +c,c,e,c,a,a,d,c,d +c,,e,c,a,a,d,c,d +a,c,e,a,a,d,d,b,b +c,a,f,b,a,,f,a,a +a,,e,b,a,d,c,b,b +b,c,f,b,a,a,e,b,a +b,a,f,b,a,a,d,b,a +a,c,e,c,a,,d,d,d +a,d,g,d,a,,d,c,c +d,d,e,c,a,,f,c,b +a,b,e,c,a,c,f,a,a +c,c,e,c,a,c,d,c,a +c,a,e,c,a,c,d,b,d +a,a,f,a,a,a,d,a,d +b,c,f,b,a,,d,a,a +d,d,e,d,a,,f,d,d +b,b,e,b,a,a,f,b,a +b,b,e,a,a,a,e,b,b +b,b,f,b,a,d,c,a,a +a,c,e,b,a,,c,c,c +b,b,f,b,a,b,e,b,a +a,a,e,c,a,d,c,c,c +c,c,f,c,a,,d,c,a +d,d,f,c,a,d,d,d,b +d,c,e,d,a,d,d,d,d +d,d,f,d,a,b,d,d,d +d,a,f,c,a,c,d,c,c +d,c,e,d,a,d,d,d,c +d,c,e,d,a,b,c,c,b +b,a,e,b,a,,c,b,a +c,d,e,b,a,b,d,c,a +c,d,f,b,a,d,c,c,d +a,a,f,c,a,,d,c,a +a,,e,a,a,b,e,a,d +a,b,f,b,a,a,d,b,c +c,d,e,d,a,b,c,c,d +a,c,e,c,a,b,b,d,a +c,c,d,c,a,b,c,c,a +b,b,f,b,a,,e,b,a +d,c,e,d,a,d,d,d,d +a,c,e,a,a,d,d,b,d +a,b,e,d,a,d,d,d,b +a,b,f,a,a,d,d,d,b +d,a,e,c,a,,c,d,d +d,c,e,d,a,b,c,d,d +a,d,e,c,a,,c,c,d +d,d,f,d,a,c,e,d,d +d,d,g,d,a,b,c,d,c +b,a,f,a,a,d,d,b,a +c,a,e,c,a,c,c,c,b +c,c,e,c,a,b,e,d,d +b,a,f,b,a,,c,b,a +d,d,e,d,a,d,d,d,d +d,d,e,d,a,b,c,d,b +b,b,f,b,a,d,c,b,a +c,d,e,d,a,b,c,c,d +b,a,f,b,a,,d,b,d +b,b,i,b,a,a,f,b,a +a,a,f,a,a,a,c,a,a +d,d,e,c,a,c,e,d,a +b,b,h,b,a,c,d,a,a +b,a,e,b,a,c,d,a,a +b,c,e,b,a,b,d,a,b +c,c,e,d,a,b,e,c,c +b,b,f,b,a,a,c,b,a +c,a,e,a,a,,d,a,d +c,d,f,d,a,,e,c,b +a,d,f,c,a,d,c,a,b +b,a,e,b,a,a,e,b,c +b,a,f,b,a,a,c,b,b +a,c,e,c,a,d,e,b,b +a,,d,a,a,a,e,a,b +c,c,e,c,a,b,c,d,d +a,b,e,a,a,a,c,a,b +b,b,e,a,a,b,b,a,a +d,d,e,d,a,c,c,d,a +b,b,f,b,a,a,d,b,c +a,a,e,b,a,,e,a,d +a,b,f,a,a,c,e,a,a +b,b,e,b,a,,c,b,a +c,a,e,a,a,d,c,c,b +b,b,e,b,a,,e,b,c +c,c,e,a,a,b,c,c,c +d,d,e,d,a,,c,d,c +a,d,f,a,a,,d,a,d +b,b,f,b,a,,d,b,a +d,d,e,d,a,c,c,d,b +b,c,f,b,a,c,c,b,c +d,c,f,c,a,b,d,c,a +c,a,f,c,a,d,e,a,a +b,b,h,b,a,,d,b,a +d,d,f,d,a,c,d,d,a +d,c,f,c,a,,f,d,d +a,,f,d,a,a,c,d,c +a,d,f,d,a,,c,c,c +a,,f,a,a,d,d,b,a +b,,f,b,a,b,d,b,a +b,a,e,a,a,d,c,b,d +d,d,e,d,a,c,f,d,d +a,b,e,b,a,d,c,b,a +a,a,e,a,a,c,c,a,a +a,b,e,a,a,c,d,a,a +c,a,e,a,a,c,d,c,b +b,b,e,b,a,a,d,b,b +d,c,f,c,a,,c,c,b +a,,e,a,a,,f,b,a +b,b,f,b,a,d,d,b,a +b,c,f,a,a,,d,b,a +d,d,f,d,a,b,d,d,b +a,,e,d,a,,c,d,d +d,,e,d,a,d,d,d,d +b,a,e,b,a,,c,b,a +d,,e,c,a,,d,c,a +a,b,e,b,a,b,c,b,a +c,a,e,c,a,d,c,d,c +a,b,e,c,a,d,d,b,c +a,a,e,a,a,d,d,a,b +c,a,f,a,a,,e,a,b +c,,e,c,a,,d,c,b +c,,e,d,a,b,d,a,b +c,d,f,a,a,b,d,d,c +b,b,f,b,a,a,d,b,a +c,a,e,c,a,,d,a,d +c,c,f,c,a,c,d,c,a +d,d,e,d,a,b,c,d,d +a,c,e,c,a,c,d,a,c +c,d,e,c,a,,c,c,b +c,c,f,d,a,c,d,c,b +c,,e,c,a,,c,c,a +d,d,e,d,a,,d,d,d +c,c,f,b,a,,e,a,a +d,c,e,d,a,,f,d,c +a,c,e,a,a,b,c,c,a +a,,e,c,a,,c,c,d +a,a,d,a,a,d,e,a,c +b,b,f,b,a,b,c,b,a +a,c,e,a,a,b,d,c,a +a,a,e,a,a,b,b,a,a +d,d,e,d,a,b,c,d,d +a,a,e,a,a,,c,b,d +a,c,e,c,a,d,e,c,c +c,d,e,c,a,,d,a,a +c,c,e,c,a,b,f,c,c +d,d,e,d,a,b,e,d,d +b,b,f,b,a,d,c,b,a +a,,e,a,a,c,e,b,a +b,b,f,a,a,d,e,a,b +d,d,e,d,a,c,d,d,c +a,a,e,b,a,,d,b,a +b,b,e,b,a,a,e,a,a +a,b,e,c,a,d,e,c,a +a,c,e,a,a,a,c,c,b +a,,e,a,a,a,d,a,a +d,d,f,d,a,,d,d,d +c,a,f,a,a,,d,a,a +a,a,f,a,a,,d,a,a +c,a,e,c,a,,b,d,b +c,b,f,a,a,a,d,a,a +c,,f,c,a,d,d,c,b +b,,f,a,a,c,d,c,c +c,a,e,a,a,d,c,c,b +b,,f,b,a,a,c,b,b +b,,e,b,a,b,c,b,a +a,d,f,c,a,,e,a,b +c,,e,d,a,a,b,a,c +d,c,e,d,a,c,d,c,d +d,,e,d,a,b,d,b,d +b,c,e,c,a,,c,a,c +d,d,e,c,a,b,e,c,a +a,c,e,a,a,d,d,a,d +c,c,e,c,a,,c,d,a +b,b,f,b,a,a,e,b,a +d,c,e,c,a,,d,c,c +a,d,f,a,a,a,f,c,d +a,d,e,c,a,d,d,c,d +c,c,d,c,a,c,e,d,d +c,c,e,a,a,d,d,a,d +c,d,e,c,a,,d,d,d +a,c,e,a,a,c,e,a,a +b,c,e,b,a,,c,a,c +d,d,f,d,a,b,c,d,b +c,c,e,d,a,c,d,c,c +d,d,e,d,a,c,d,d,b +d,d,f,d,a,c,d,c,d +d,c,e,d,a,d,c,c,d +c,d,e,c,a,,c,d,d +b,a,g,a,a,a,c,c,a +d,d,f,c,a,d,c,c,d +c,d,e,c,a,,c,b,d +a,c,e,c,a,b,c,c,b +b,,e,b,a,,e,b,b +d,c,e,d,a,,d,d,d +b,c,e,a,a,d,e,b,a +d,,f,a,a,,c,c,a +c,d,e,c,a,c,e,d,b +d,d,e,d,a,d,b,d,d +d,d,e,d,a,c,c,d,d +c,c,e,a,a,b,d,c,a +d,c,f,d,a,b,d,d,b +d,c,e,d,a,,c,d,b +d,d,e,d,a,,c,d,d +d,d,f,d,a,c,e,d,b +a,b,e,b,a,a,d,a,a +b,,f,b,a,a,c,b,c +a,a,e,a,a,a,c,b,b +a,b,f,a,a,,c,a,c +b,b,e,a,a,a,e,a,d +a,d,e,b,a,,e,a,b +a,c,e,a,a,c,c,c,a +b,,e,b,a,c,d,b,b +d,,e,c,a,d,c,c,b +a,c,e,c,a,b,d,c,b +c,a,g,d,a,,e,c,c +d,,e,a,a,a,c,d,d +a,a,e,a,a,,c,b,c +b,b,i,a,a,,e,a,a +d,d,e,c,a,c,d,c,c +a,c,f,c,a,,d,a,b +c,a,f,a,a,b,d,c,a +d,d,e,d,a,,c,d,c +b,b,e,b,a,d,d,c,c +d,d,e,d,a,b,c,d,c +c,d,f,c,a,b,c,a,c +d,a,e,a,a,a,d,d,d +d,c,e,d,a,a,e,d,d +b,b,e,b,a,d,e,b,a +c,c,d,a,a,,f,a,a +a,c,e,a,a,b,d,a,a +d,d,e,d,a,,c,d,c +a,b,h,b,a,,d,b,a +b,a,e,a,a,b,d,a,d +b,a,f,a,a,b,d,a,a +c,d,e,d,a,d,b,d,d +b,a,f,b,a,a,d,b,a +d,c,e,c,a,c,e,a,a +d,d,e,d,a,,e,c,a +d,d,e,d,a,b,c,d,d +b,b,e,a,a,,b,b,b +d,d,e,c,a,c,e,d,b +c,c,e,c,a,a,d,c,a +d,c,e,d,a,a,c,c,d +a,a,e,c,a,d,d,a,d +c,,e,d,a,,e,c,b +b,b,d,b,a,,c,b,a +a,c,e,a,a,c,d,a,a +d,d,e,d,a,c,c,d,d +d,d,e,d,a,d,c,d,d +c,c,e,c,a,c,b,c,d +c,b,e,a,a,,c,c,c +c,a,e,c,a,c,d,a,a +a,b,e,a,a,a,e,a,d +b,b,e,b,a,d,d,b,c +d,d,e,d,a,c,e,d,b +c,d,e,d,a,d,d,d,b +c,c,e,a,a,,d,c,b +c,b,e,c,a,,d,a,a +a,c,e,a,a,b,c,a,d +a,b,e,c,a,b,c,c,b +a,a,f,b,a,b,c,b,b +a,,e,c,a,,d,a,a +a,d,e,b,a,a,c,a,c +c,d,f,c,a,,c,c,c +d,d,e,d,a,,e,d,b +b,d,e,a,a,,c,a,a +a,a,f,a,a,,e,a,a +c,a,e,a,a,d,d,a,a +d,c,e,d,a,b,d,d,d +c,d,d,c,a,,d,d,d +d,d,e,d,a,c,d,d,d +b,b,e,b,a,a,d,b,c +c,d,e,c,a,a,e,a,b +c,c,e,c,a,d,c,c,d +b,b,e,b,a,a,d,b,a +d,d,f,d,a,b,e,d,d +b,c,h,a,a,,d,b,d +a,a,e,a,a,,d,b,a +c,,f,d,a,b,c,c,d +b,b,e,b,a,b,c,b,a +b,b,f,b,a,d,d,b,a +c,a,f,c,a,c,d,a,d +c,b,f,a,a,a,d,a,a +c,d,f,c,a,b,c,c,b +c,,e,d,a,,e,d,b +b,b,e,b,a,,d,b,a +a,,e,b,a,d,c,a,b +a,c,e,c,a,b,f,d,a +a,,f,b,a,a,d,b,d +c,c,e,c,a,b,d,d,d +d,d,f,d,a,c,d,d,d +d,d,e,d,a,b,d,d,d +c,b,d,c,a,a,d,a,a +b,a,d,b,a,b,b,b,a +b,b,e,b,a,a,c,b,a +c,d,e,d,a,,e,d,d +b,b,e,b,a,,e,b,c +c,b,f,a,a,b,d,a,a +c,a,e,d,a,a,b,c,a +d,d,f,d,a,,f,d,b +a,a,e,c,a,b,d,b,a +c,,e,d,a,,d,c,a +c,d,e,d,a,d,c,d,d +a,,e,c,a,c,c,c,b +d,a,e,d,a,b,d,d,d +c,d,e,a,a,a,d,b,d +c,c,e,a,a,c,e,c,b +a,b,f,b,a,a,e,a,a +a,a,f,c,a,c,d,c,a +a,a,e,c,a,b,c,c,d +c,d,e,c,a,b,e,d,b +a,b,e,a,a,d,e,a,a +c,b,e,a,a,c,c,c,c +c,d,f,c,a,,d,a,b +c,a,d,d,a,a,d,a,b +a,b,f,b,a,a,e,c,a +a,b,f,a,a,a,e,a,c +d,d,f,d,a,b,e,d,d +c,d,f,c,a,d,d,b,b +a,a,e,b,a,a,d,c,b +b,b,e,b,a,d,c,b,c +a,,e,b,a,a,c,c,d +a,,f,b,a,,d,a,b +c,d,f,c,a,d,d,c,d +d,c,e,d,a,d,c,d,d +a,c,e,c,a,,d,c,a +a,b,e,b,a,a,d,c,a +a,a,f,b,a,d,d,b,b +c,d,f,d,a,c,c,c,b +d,a,e,b,a,c,d,a,a +a,a,e,a,a,,c,c,b +c,c,e,c,a,b,c,a,d +c,d,f,a,a,d,e,c,b +b,b,e,b,a,a,d,b,c +a,a,f,a,a,,c,a,a +c,d,e,c,a,,c,d,d +b,b,e,b,a,d,d,b,a +c,,e,c,a,a,d,a,d +c,,e,d,a,b,f,c,c +b,a,e,a,a,b,d,b,a +a,c,e,c,a,d,d,a,b +c,c,f,c,a,,d,a,d +c,a,f,a,a,,c,a,a +a,c,e,a,a,b,d,c,d +c,b,e,a,a,d,d,a,c +c,d,e,c,a,,d,d,d +a,,e,a,a,b,d,c,b +c,a,f,b,a,b,e,c,a +b,a,e,a,a,,c,a,c +d,a,e,d,a,b,d,d,a +b,a,e,c,a,,b,c,d +c,c,e,a,a,b,d,a,a +c,c,e,c,a,b,c,d,b +d,c,e,a,a,,d,c,b +b,b,f,b,a,a,d,b,b +b,b,e,a,a,a,e,a,c +b,a,g,b,a,,c,b,a +d,d,e,d,a,c,c,d,b +a,,e,c,a,c,d,c,a +a,c,f,c,a,d,c,b,b +c,d,e,a,a,,d,a,c +b,b,e,b,a,b,d,b,a +b,c,e,b,a,c,d,b,a +d,,e,d,a,b,d,d,b +a,a,e,a,a,d,c,a,b +a,c,f,a,a,,e,b,a +d,a,f,d,a,,d,d,d +b,b,g,a,a,,d,b,b +d,c,e,d,a,,c,d,d +c,c,f,a,a,b,d,c,c +d,,e,d,a,,e,d,d +c,a,e,d,a,b,e,a,b +c,d,f,c,a,d,d,b,a +c,a,e,c,a,d,d,c,d +c,a,f,c,a,b,e,b,c +d,,e,d,a,c,e,c,d +c,a,e,c,a,,d,a,b +b,b,e,b,a,d,e,b,a +d,d,e,d,a,c,e,d,b +a,d,e,c,a,c,b,d,b +c,c,e,a,a,,d,c,c +a,a,e,a,a,a,c,a,a +b,b,e,b,a,,f,a,a +c,d,e,a,a,b,e,d,a +a,c,e,a,a,a,c,d,c +d,d,d,d,a,b,c,d,d +d,d,e,d,a,b,e,d,d +c,d,e,a,a,,c,c,c +d,d,e,d,a,c,d,d,d +a,b,e,a,a,,d,c,a +c,b,e,a,a,a,c,c,b +a,a,e,d,a,,d,a,b +c,d,f,c,a,c,e,d,a +c,d,e,b,a,c,c,a,a +d,d,e,d,a,,d,d,b +c,c,e,a,a,,d,a,c +b,c,e,c,a,,c,a,d +c,c,f,c,a,a,d,c,d +d,c,f,d,a,b,d,c,d +b,a,e,b,a,a,d,b,a +c,c,e,c,a,c,c,d,c +d,d,e,d,a,c,b,d,a +b,a,e,b,a,b,d,b,a +b,b,f,b,a,a,b,b,a +c,b,f,a,a,b,c,a,a +a,,e,a,a,d,d,c,d +d,c,e,a,a,c,c,d,c +c,c,e,a,a,b,d,d,c +a,d,e,c,a,d,d,c,c +a,c,e,a,a,a,c,a,a +b,b,f,a,a,b,c,a,a +c,a,f,d,a,,d,d,c +c,,e,d,a,,d,d,b +a,a,e,a,a,a,d,a,a +d,,e,c,a,,e,c,b +a,c,e,a,a,,e,a,a +d,a,f,a,a,b,c,c,c +a,c,e,c,a,b,c,a,d +c,d,f,a,a,a,d,c,d +d,c,e,c,a,b,d,d,d +c,a,e,c,a,,d,c,b +a,a,f,b,a,a,c,a,a +c,,f,c,a,b,f,c,d +c,d,e,b,a,b,d,c,b +d,d,e,d,a,c,d,c,d +c,a,e,d,a,d,e,d,d +b,b,f,c,a,,e,b,b +b,a,d,a,a,d,d,a,a +a,a,f,c,a,,e,a,b +b,b,f,b,a,c,d,b,a +b,b,e,b,a,d,e,b,a +d,a,e,c,a,,e,c,b +a,a,e,b,a,a,e,b,a +b,b,f,b,a,d,d,b,a +a,a,e,b,a,d,c,a,c +b,a,e,a,a,c,d,b,a +a,,f,a,a,a,d,a,a +d,c,f,c,a,c,d,c,a +a,b,e,a,a,d,d,c,a +a,b,e,b,a,,c,b,d +a,a,f,a,a,a,e,a,b +b,b,e,b,a,a,e,b,a +d,d,e,c,a,d,d,d,d +a,a,f,a,a,,c,a,a +d,c,e,c,a,a,d,c,d +c,c,e,c,a,,d,c,a +c,c,e,c,a,b,f,c,d +d,d,e,c,a,,d,a,b +b,b,e,b,a,,d,b,b +b,c,f,a,a,,e,c,a +a,c,e,c,a,,e,d,b +b,b,e,b,a,,e,b,a +d,d,e,d,a,c,d,d,b +b,b,f,b,a,b,c,b,a +c,,d,d,a,c,c,d,d +b,b,f,b,a,a,d,b,a +a,,f,c,a,b,d,a,b +d,d,e,d,a,b,e,d,d +c,,e,d,a,d,e,d,b +b,b,d,b,a,d,c,b,a +a,c,e,c,a,c,d,c,d +c,a,e,c,a,a,c,c,b +a,a,e,c,a,a,f,a,a +c,b,e,c,a,a,d,a,c +a,d,f,c,a,,e,c,c +b,a,f,b,a,a,c,b,a +b,c,f,b,a,a,c,c,a diff --git a/data/synthetic-data-gan.csv b/data/synthetic-data-gan.csv new file mode 100644 index 0000000..48944ca --- /dev/null +++ b/data/synthetic-data-gan.csv @@ -0,0 +1,1001 @@ +Total_population__Both_sexes_,Taxable_income__million_yen_,Ordinal_Number_of_employed_persons,Number_of_households__Total_,Ordinal_Number_of_physicians,Number_of_dwellings_with_occupying_households,Ordinal_Total_population__65_and_over_,Number_of_lower_secondary_school_students,Number_of_elderly_nursing_facilities__Questionnaire_about_basic_matters_ +d,a,f,d,a,,c,a,a +c,c,e,c,a,,d,c,a +c,,f,c,a,,d,c,b +c,b,e,d,a,a,d,a,c +d,a,e,d,a,d,e,a,d +b,a,d,c,a,a,e,d,d +c,d,f,a,a,,d,b,b +d,,f,d,a,c,c,a,d +d,d,e,a,a,,c,d,a +c,d,f,c,a,c,c,b,b +b,a,f,a,a,a,d,b,d +b,d,e,c,a,d,c,b,b +d,d,e,c,a,d,c,d,d +b,a,e,a,a,,d,a,a +c,d,e,c,a,a,b,d,d +c,d,e,a,a,c,d,c,b +a,,f,c,a,d,c,a,a +b,b,e,b,a,,d,b,d +c,b,e,a,a,,e,a,a +a,a,e,a,a,,d,a,b +c,c,f,d,a,b,d,b,b +d,c,e,b,a,a,c,c,d +b,a,e,d,a,b,c,a,c +c,,d,c,a,b,d,d,c +d,c,g,d,a,a,d,c,c +c,c,e,a,a,d,d,c,b +a,d,e,d,a,c,c,d,d +d,d,e,d,a,d,c,d,d +b,b,f,a,a,b,e,b,a +b,b,f,c,a,b,b,b,b +c,,f,b,a,,d,c,a +b,b,f,b,a,a,c,c,b +b,c,e,c,a,b,c,b,d +a,b,e,c,a,b,d,d,d +c,b,i,b,a,c,e,a,d +b,d,e,d,a,d,c,b,d +c,c,f,c,a,b,c,d,c +b,c,f,b,a,a,e,a,a +c,c,g,b,a,b,d,c,c +b,b,e,b,a,d,d,a,b +d,b,e,b,a,,c,d,c +b,c,h,b,a,a,d,b,c +b,,e,c,a,c,e,d,a +d,b,e,d,a,d,c,c,d +c,a,e,c,a,b,e,b,a +b,a,f,b,a,,d,a,c +c,a,f,c,a,c,d,d,b +d,d,e,b,a,d,c,c,b +c,b,f,a,a,c,f,d,a +a,c,i,d,a,d,d,d,b +d,b,e,d,a,d,d,c,b +a,b,f,b,a,a,d,c,d +b,d,h,d,a,d,e,b,d +b,a,f,b,a,,d,b,a +c,b,e,b,a,b,d,c,d +c,c,e,d,a,d,c,b,c +d,c,f,b,a,,d,c,c +a,d,e,a,a,,d,a,a +b,c,f,c,a,d,e,b,c +b,b,e,b,a,d,c,d,c +d,c,e,a,a,d,d,d,c +c,b,e,c,a,d,e,a,b +c,a,d,c,a,,e,b,b +a,,e,c,a,c,c,b,d +b,c,e,b,a,a,d,b,c +d,c,e,b,a,d,d,b,d +d,d,e,c,a,a,d,a,d +c,b,e,c,a,c,d,a,c +d,d,e,d,a,d,c,d,d +c,d,e,d,a,d,d,d,b +d,d,d,a,a,d,f,a,c +b,b,e,d,a,d,d,d,d +c,,e,c,a,,f,c,d +a,c,e,b,a,,d,d,d +b,a,e,d,a,b,d,b,a +d,d,d,c,a,c,d,c,b +c,a,f,c,a,a,d,c,a +c,b,f,a,a,,f,a,b +c,b,e,b,a,b,c,c,b +a,a,f,a,a,,d,a,b +d,d,f,b,a,d,c,b,d +a,a,e,a,a,b,d,a,c +a,b,f,a,a,,e,b,c +c,c,e,b,a,a,e,c,c +b,a,f,a,a,a,d,c,b +c,,e,c,a,b,c,c,d +b,b,f,a,a,,d,b,a +c,d,e,d,a,c,e,d,d +c,c,e,c,a,a,e,c,c +b,b,f,c,a,a,d,a,b +d,a,e,b,a,,d,b,a +d,b,d,b,a,a,b,d,b +c,c,e,c,a,c,c,a,a +b,c,f,c,a,,e,c,b +d,b,d,c,a,d,e,a,b +a,b,f,a,a,a,d,a,b +c,d,e,d,a,d,d,d,c +a,,f,b,a,b,d,a,c +d,d,e,d,a,d,c,c,d +d,a,d,c,a,b,d,d,a +c,d,e,d,a,c,d,b,b +d,c,e,b,a,,d,a,a +c,,e,c,a,a,c,b,d +d,,e,d,a,c,d,d,d +d,d,e,d,a,d,c,d,d +b,b,f,d,a,a,d,d,a +c,,e,c,a,b,d,c,b +d,d,e,c,a,d,c,c,c +a,a,e,c,a,d,d,c,d +b,d,e,b,a,,c,d,b +a,b,f,d,a,a,e,a,a +d,d,f,b,a,,d,c,b +b,b,e,a,a,,d,b,a +a,d,e,d,a,d,c,c,d +c,c,e,a,a,d,c,c,b +a,b,f,c,a,,c,c,b +b,d,e,c,a,d,c,c,d +c,c,e,d,a,b,e,a,c +b,b,e,a,a,,d,a,a +b,c,f,d,a,a,d,c,a +c,b,e,c,a,,d,b,b +b,c,e,c,a,c,c,a,a +d,c,e,c,a,b,e,d,d +a,b,i,b,a,b,g,b,c +d,d,e,d,a,,f,d,d +d,b,e,d,a,,e,b,a +c,c,e,d,a,d,e,b,b +a,d,h,b,a,c,d,b,d +d,d,e,d,a,c,c,d,d +c,,e,c,a,c,d,b,b +a,c,e,c,a,c,c,c,c +a,c,f,b,a,a,e,b,b +b,c,e,d,a,b,d,b,b +b,c,g,c,a,c,c,d,c +a,a,d,a,a,c,f,c,b +d,d,e,c,a,d,d,c,d +b,,e,b,a,b,d,a,c +c,,e,d,a,,e,d,b +b,a,f,b,a,,d,b,a +c,b,e,b,a,b,d,b,d +b,d,e,a,a,b,d,c,c +a,a,h,b,a,a,e,c,a +c,c,g,b,a,,d,c,c +d,d,e,c,a,,d,d,b +d,d,e,c,a,c,c,d,d +a,c,e,c,a,b,e,a,b +c,b,e,c,a,c,g,c,d +b,c,e,d,a,,c,c,a +b,c,e,c,a,d,c,c,b +c,a,e,a,a,,d,a,a +c,b,e,c,a,d,d,b,c +c,c,e,d,a,b,d,b,b +b,c,e,b,a,b,c,c,b +a,b,e,d,a,,c,a,b +d,c,e,a,a,,f,d,b +b,c,f,a,a,a,d,c,a +c,,e,b,a,c,f,b,d +c,,e,b,a,c,c,c,b +b,d,e,d,a,d,c,d,c +b,a,e,b,a,,g,d,d +b,a,f,a,a,b,d,b,b +a,d,f,c,a,,d,a,b +c,c,e,c,a,a,c,c,b +a,a,f,a,a,a,d,a,a +b,b,e,d,a,,c,b,d +b,a,e,b,a,a,c,a,b +c,d,e,d,a,b,c,d,d +c,d,e,c,a,d,d,b,c +d,c,e,d,a,c,c,c,d +a,b,e,c,a,b,e,b,b +c,a,e,c,a,,d,c,a +d,c,e,d,a,b,b,d,a +d,a,e,d,a,c,c,b,d +c,a,d,d,a,,f,b,a +a,a,f,a,a,,d,b,a +d,b,e,b,a,,c,b,a +c,b,e,b,a,a,c,c,b +b,d,e,a,a,,d,a,a +b,a,f,b,a,,d,b,a +d,c,e,b,a,a,d,c,b +b,,e,b,a,d,d,c,c +b,d,f,c,a,b,c,c,b +a,c,f,b,a,,c,a,a +d,d,e,c,a,,c,c,c +d,c,d,b,a,a,f,d,a +c,a,f,a,a,d,e,a,b +d,c,e,a,a,d,c,d,d +d,c,e,d,a,,d,b,c +d,b,e,d,a,a,d,a,c +d,c,e,d,a,d,c,d,d +a,c,e,c,a,,c,a,a +a,c,e,c,a,d,e,c,b +d,d,e,a,a,c,e,b,c +d,b,e,b,a,d,d,b,d +b,d,f,b,a,,d,b,b +a,d,e,a,a,a,b,a,a +c,c,f,b,a,b,d,b,a +a,c,f,d,a,d,d,a,a +b,c,e,d,a,,d,a,d +c,c,e,c,a,b,d,d,d +a,d,f,d,a,a,e,a,b +b,,e,c,a,d,c,c,d +c,d,g,d,a,d,b,c,c +b,b,f,b,a,,d,b,b +b,b,e,a,a,,e,a,b +d,c,e,a,a,b,c,d,c +b,a,e,a,a,,c,b,c +b,b,f,b,a,a,e,b,a +c,b,e,c,a,b,c,c,a +c,a,f,b,a,,d,b,a +a,b,f,a,a,a,d,a,a +b,b,e,b,a,a,c,b,c +a,,e,d,a,b,d,c,b +c,a,e,b,a,,d,b,c +c,b,e,a,a,,e,b,c +b,c,e,b,a,a,e,b,b +c,c,e,b,a,b,d,a,a +a,b,e,d,a,,e,b,a +d,c,e,d,a,b,c,b,c +b,c,f,d,a,b,c,d,c +b,c,f,b,a,b,g,d,a +d,d,e,a,a,a,d,a,d +a,a,f,a,a,,d,a,a +a,c,e,a,a,c,d,a,c +c,d,f,c,a,b,c,c,b +b,c,f,a,a,c,c,c,c +d,b,f,d,a,d,c,c,d +c,a,e,b,a,b,e,a,b +b,b,e,b,a,c,d,b,d +c,b,e,c,a,a,c,b,a +b,b,d,a,a,,e,c,a +b,c,e,b,a,,g,b,c +b,a,e,b,a,d,d,b,c +b,c,e,b,a,,d,c,a +b,d,e,c,a,b,c,c,c +a,d,e,b,a,b,d,b,a +b,d,e,d,a,d,d,b,c +d,d,e,c,a,b,c,c,b +c,c,e,b,a,b,d,d,b +c,d,e,a,a,b,c,c,c +c,c,d,b,a,b,d,c,c +d,d,g,d,a,d,b,d,d +d,d,e,c,a,d,c,d,a +c,c,e,c,a,b,c,c,b +a,a,f,a,a,b,d,c,d +b,b,e,b,a,d,c,b,d +d,,e,d,a,c,c,d,d +b,a,h,c,a,,c,c,a +a,a,f,a,a,,e,a,a +d,d,e,b,a,c,f,c,d +c,d,f,b,a,,e,a,d +c,d,e,a,a,d,c,c,c +b,b,e,a,a,,e,a,c +b,b,f,b,a,c,d,b,a +c,b,f,a,a,,d,a,c +b,c,e,b,a,c,f,b,a +d,c,e,b,a,c,c,d,d +b,b,f,b,a,,e,a,c +c,a,f,a,a,a,c,a,a +a,d,f,d,a,b,c,c,d +c,c,e,b,a,c,d,c,a +b,c,f,c,a,b,c,a,c +b,a,f,c,a,b,d,a,a +b,b,e,b,a,a,e,b,a +c,,f,c,a,d,b,d,b +d,d,e,c,a,,d,c,b +c,c,e,d,a,c,c,a,a +c,b,e,b,a,a,c,d,c +a,d,e,d,a,,d,a,c +a,a,e,b,a,,d,a,d +a,d,e,d,a,d,c,d,c +a,a,f,a,a,,e,b,a +a,a,f,d,a,c,d,d,a +a,a,e,b,a,c,e,a,d +b,a,e,b,a,b,d,d,d +b,a,e,b,a,,d,b,b +d,,e,d,a,b,b,b,c +d,b,f,b,a,a,d,b,a +c,b,e,c,a,a,c,d,c +b,a,e,d,a,c,d,d,d +b,d,d,d,a,a,f,a,c +b,a,f,a,a,a,c,c,a +d,d,h,d,a,d,c,d,d +b,b,f,b,a,,d,b,b +c,d,e,b,a,b,d,b,d +d,c,e,d,a,d,e,c,c +a,a,f,b,a,a,d,a,a +b,c,e,b,a,d,c,c,b +c,c,e,d,a,c,d,b,b +c,b,e,c,a,,d,b,b +a,,e,b,a,a,c,c,a +b,c,e,d,a,c,c,c,d +c,,e,a,a,,d,b,c +c,c,e,d,a,c,d,a,c +c,c,e,c,a,b,e,b,b +d,,e,d,a,,c,a,a +d,a,f,d,a,d,d,b,c +d,a,e,b,a,,e,b,b +c,c,e,b,a,b,e,c,b +a,c,f,a,a,,d,c,a +b,a,e,b,a,,e,b,d +a,c,f,b,a,b,d,c,b +d,d,e,c,a,c,c,c,b +a,a,f,a,a,,d,b,b +c,d,f,a,a,a,e,d,b +b,b,e,c,a,b,d,a,c +d,,e,d,a,d,c,d,c +a,b,e,b,a,b,c,c,d +c,b,f,d,a,a,f,b,b +b,a,f,b,a,,e,b,a +c,b,e,c,a,d,c,d,d +c,d,e,a,a,c,c,b,d +c,,e,c,a,a,d,d,d +d,a,e,b,a,d,d,b,a +c,c,h,b,a,a,b,a,a +c,b,e,c,a,a,c,c,d +d,c,e,b,a,a,d,c,b +d,a,e,d,a,d,d,a,b +b,c,e,d,a,a,c,d,d +c,d,e,d,a,d,d,c,c +c,c,f,b,a,d,d,d,c +d,c,e,a,a,,d,d,a +d,b,e,b,a,,e,d,b +d,,e,c,a,d,c,c,b +b,,e,c,a,,d,b,a +c,a,f,a,a,,e,b,a +d,,e,d,a,d,f,d,d +c,b,f,c,a,b,c,b,b +d,d,e,c,a,c,e,a,d +b,a,e,d,a,,c,d,c +b,b,f,d,a,d,d,c,d +a,a,f,a,a,b,d,a,b +b,a,e,b,a,d,b,a,c +b,c,e,b,a,a,c,d,d +c,b,g,c,a,c,d,c,d +b,d,e,d,a,d,c,b,d +d,c,e,c,a,b,c,c,d +a,b,e,b,a,,d,c,b +c,d,e,d,a,c,c,d,d +c,d,e,c,a,b,d,c,d +c,c,f,c,a,b,d,b,a +d,d,e,d,a,a,c,d,d +c,,e,c,a,c,d,b,b +a,a,e,b,a,a,c,b,a +d,c,e,c,a,b,d,c,b +b,b,e,b,a,a,d,c,a +a,a,e,c,a,,d,b,c +b,b,d,b,a,d,d,b,d +d,d,d,d,a,a,d,c,a +a,a,e,d,a,,c,a,a +a,b,f,c,a,,e,b,a +c,,f,d,a,a,b,b,a +b,a,e,b,a,a,d,b,a +d,b,e,d,a,,e,d,d +b,b,e,b,a,a,c,d,b +b,d,e,d,a,a,c,a,c +b,c,f,a,a,a,d,a,a +d,c,e,c,a,a,c,c,c +c,d,i,a,a,c,d,c,b +a,c,f,b,a,a,e,b,b +d,d,g,c,a,b,d,b,d +c,b,e,d,a,a,c,d,a +b,b,e,b,a,b,d,c,b +d,c,e,d,a,c,c,c,d +b,d,e,d,a,a,c,b,b +c,c,e,d,a,,c,c,a +c,a,f,a,a,a,d,a,d +b,c,e,a,a,,d,d,d +a,,f,c,a,,e,d,c +a,d,e,a,a,,d,c,b +c,c,f,c,a,a,c,c,a +c,d,e,b,a,d,c,d,b +d,d,e,b,a,a,d,a,d +d,c,f,b,a,b,d,c,a +b,a,e,c,a,a,c,a,c +b,a,f,b,a,a,d,a,c +c,b,f,a,a,,d,a,c +a,b,e,b,a,c,c,d,b +b,,e,b,a,a,e,b,d +b,a,f,c,a,a,d,a,a +b,d,e,b,a,,c,d,b +c,c,e,d,a,b,c,c,d +c,c,e,c,a,c,d,c,c +b,d,e,c,a,a,d,c,b +c,c,e,b,a,b,e,c,a +c,d,e,c,a,d,c,c,b +c,d,e,d,a,d,c,c,d +a,a,e,a,a,d,e,a,a +c,b,e,a,a,,d,c,a +d,,e,b,a,c,g,d,c +a,c,e,c,a,a,c,d,b +a,c,e,b,a,c,c,d,a +c,d,e,c,a,a,c,a,b +c,a,e,d,a,c,g,b,d +d,d,e,c,a,d,d,d,b +d,d,e,d,a,a,c,d,d +d,d,e,b,a,d,e,b,b +b,b,e,b,a,c,c,b,c +d,d,e,a,a,d,d,b,a +a,a,f,d,a,a,f,c,a +c,a,e,c,a,,e,c,c +a,d,e,d,a,d,e,c,b +d,b,f,d,a,a,d,a,b +d,,f,d,a,b,d,c,c +c,,e,d,a,c,d,d,c +a,d,f,a,a,,d,a,b +c,d,e,c,a,b,c,c,d +d,b,f,b,a,,g,b,a +b,c,e,c,a,d,b,c,b +c,b,g,c,a,c,d,d,b +b,b,f,b,a,,e,a,b +b,b,f,b,a,,d,c,b +c,d,e,d,a,d,c,d,d +b,d,e,a,a,d,d,d,c +d,b,e,c,a,,c,b,a +c,c,e,d,a,c,c,b,b +b,a,f,a,a,,d,a,c +c,d,e,b,a,,f,b,a +c,a,e,c,a,b,d,c,b +a,a,d,a,a,,d,a,a +c,b,e,a,a,,d,c,a +b,,f,a,a,,f,b,a +b,b,e,c,a,,d,a,b +c,a,e,d,a,,d,a,d +d,b,e,d,a,a,f,d,b +c,d,e,c,a,c,d,d,c +b,b,e,b,a,a,e,d,b +a,,h,b,a,c,g,d,c +a,b,e,c,a,,c,c,c +b,b,e,c,a,a,c,a,b +c,a,e,d,a,,c,b,c +d,d,e,d,a,b,c,d,c +d,c,e,b,a,a,b,b,b +d,d,e,d,a,b,f,d,d +d,c,e,c,a,c,c,c,b +c,c,e,a,a,c,d,d,a +a,d,e,a,a,a,d,b,c +c,,i,c,a,,d,a,c +c,c,e,a,a,c,d,c,d +c,a,f,a,a,,e,a,d +d,c,e,c,a,d,c,b,c +c,a,e,b,a,c,c,c,d +c,a,e,c,a,c,c,c,d +c,,g,d,a,b,c,a,a +a,,e,c,a,,c,b,a +b,b,f,b,a,,c,c,b +d,d,e,c,a,c,d,d,d +d,d,e,c,a,d,d,b,d +c,,f,b,a,d,d,c,a +b,d,d,d,a,d,d,b,d +d,d,e,c,a,b,c,c,d +d,d,e,c,a,d,d,a,d +c,d,e,d,a,,c,c,b +d,d,e,c,a,b,c,d,d +c,d,e,d,a,a,c,b,a +c,d,e,c,a,b,c,c,d +c,,e,a,a,,d,c,c +c,d,e,d,a,a,d,c,b +d,c,d,d,a,c,c,c,d +c,c,e,a,a,b,c,a,c +b,b,f,c,a,,d,b,a +a,d,e,a,a,,b,c,d +d,d,e,b,a,,c,a,d +a,a,e,d,a,c,d,c,a +b,b,e,a,a,,d,a,b +a,c,f,b,a,c,d,a,a +b,d,e,c,a,b,d,c,c +c,c,g,b,a,b,c,c,c +c,c,e,d,a,a,d,b,b +b,,f,c,a,d,d,b,c +c,a,e,b,a,,b,a,c +b,a,e,b,a,a,d,b,a +c,a,e,c,a,d,b,c,a +c,b,i,d,a,,e,b,c +b,a,f,b,a,,f,b,b +b,b,f,b,a,a,e,d,c +c,b,h,b,a,a,d,c,a +c,a,e,b,a,c,b,a,c +c,a,e,d,a,b,d,d,d +b,,f,c,a,b,g,a,b +b,d,e,a,a,,c,a,a +c,b,e,c,a,a,b,c,a +a,a,e,a,a,c,c,d,a +c,c,f,b,a,,c,b,d +c,c,e,c,a,c,d,c,d +c,b,e,c,a,b,e,b,b +d,c,e,c,a,d,c,d,a +c,c,e,b,a,b,d,d,b +c,,e,c,a,c,c,d,a +d,b,e,c,a,a,d,b,a +b,b,f,c,a,,c,c,b +c,b,g,c,a,b,b,c,c +d,d,e,c,a,b,d,c,d +b,a,f,a,a,,d,d,a +d,c,e,b,a,b,c,a,b +a,a,e,a,a,,d,b,b +b,c,e,b,a,b,d,d,c +d,d,e,d,a,c,c,d,d +c,a,f,c,a,c,d,c,d +c,c,e,b,a,d,d,a,d +b,c,e,a,a,c,d,c,b +c,a,e,c,a,a,c,c,a +c,b,f,c,a,a,d,b,b +a,c,f,a,a,,c,a,a +c,b,e,d,a,,f,a,b +b,b,e,a,a,,e,a,b +b,d,e,d,a,,d,a,a +a,c,f,b,a,,c,d,a +d,d,e,b,a,b,d,c,c +a,c,f,a,a,,d,a,b +b,b,e,b,a,d,b,c,d +b,a,f,b,a,,d,b,a +b,b,e,b,a,,d,b,a +b,a,e,a,a,d,d,a,a +c,d,g,b,a,d,c,a,a +b,a,f,d,a,c,d,b,b +b,c,f,b,a,d,f,d,c +c,c,e,d,a,c,c,d,c +b,a,f,a,a,,d,a,b +b,,e,c,a,d,c,b,b +b,c,g,c,a,b,d,c,b +c,a,e,b,a,,b,c,c +b,c,e,a,a,,d,c,b +c,d,e,b,a,c,d,b,c +c,,g,b,a,c,c,d,a +c,d,e,a,a,,d,c,a +b,c,f,d,a,c,c,b,b +b,,e,d,a,,d,b,b +d,c,e,c,a,c,b,a,b +a,c,e,b,a,a,g,d,c +d,,e,d,a,,c,d,c +b,,g,c,a,a,d,b,a +d,c,e,b,a,b,c,a,d +a,d,d,b,a,,c,c,a +b,c,e,b,a,c,d,b,c +b,b,e,d,a,,d,b,b +c,d,e,c,a,d,c,b,c +d,d,e,c,a,b,c,c,b +d,c,g,c,a,b,c,a,d +d,d,e,d,a,d,e,d,a +b,b,e,c,a,a,d,d,b +d,b,e,d,a,d,c,c,d +a,b,e,d,a,c,c,a,a +d,b,f,b,a,b,d,c,b +d,,e,c,a,a,d,d,d +d,d,e,d,a,b,c,c,b +b,a,f,b,a,,e,a,a +b,d,e,a,a,d,c,a,b +c,c,e,d,a,d,d,b,c +b,c,f,b,a,d,d,a,c +b,,i,a,a,c,d,c,b +c,d,e,b,a,c,d,c,b +b,b,i,b,a,a,d,b,b +d,b,f,d,a,d,c,d,d +a,a,e,c,a,c,e,c,d +c,c,f,b,a,b,c,b,a +b,b,e,a,a,a,b,b,a +c,,e,d,a,b,f,d,d +c,c,e,b,a,a,c,a,c +c,c,e,b,a,,d,c,a +b,d,f,a,a,,d,d,a +a,b,f,a,a,,d,c,c +d,c,e,d,a,a,c,d,c +d,c,e,d,a,c,c,d,d +c,b,e,a,a,a,d,a,a +d,d,e,d,a,a,c,d,d +b,c,e,b,a,a,e,c,b +c,,e,c,a,b,d,b,d +b,b,h,d,a,d,d,c,d +c,c,e,c,a,b,d,c,a +c,c,e,d,a,b,d,d,d +a,a,f,a,a,a,e,a,a +d,d,e,d,a,b,g,d,c +d,c,e,d,a,,d,c,c +a,d,f,b,a,b,d,c,d +c,c,e,c,a,b,d,c,c +c,c,e,d,a,,c,d,c +b,b,e,b,a,a,b,c,b +c,d,e,d,a,d,d,d,a +c,,f,a,a,d,c,b,c +d,d,e,d,a,c,c,d,d +c,d,e,b,a,,d,a,d +c,d,e,d,a,c,e,b,a +d,c,e,d,a,c,c,c,d +b,a,f,d,a,,d,c,b +c,c,e,c,a,a,d,b,c +c,b,f,a,a,a,d,a,d +c,d,f,a,a,a,b,a,a +b,d,g,a,a,c,e,b,a +c,c,d,b,a,,d,c,c +c,c,h,c,a,d,c,a,b +c,,g,a,a,b,e,a,a +a,c,e,a,a,a,b,c,d +d,d,d,a,a,c,e,c,b +a,,f,d,a,a,d,b,a +d,,e,b,a,c,c,d,d +a,c,f,b,a,a,c,a,b +c,c,e,c,a,b,c,d,c +d,c,e,d,a,d,c,b,d +d,a,e,d,a,,d,a,a +c,c,e,c,a,c,c,d,c +d,d,e,b,a,c,c,b,d +b,a,f,a,a,a,d,c,b +b,a,e,a,a,a,f,a,b +b,d,e,d,a,b,d,d,c +d,c,e,a,a,a,b,b,b +b,,f,a,a,,c,d,a +a,a,f,b,a,b,b,c,c +d,c,e,a,a,a,e,c,b +d,d,e,d,a,c,c,d,d +d,c,e,d,a,c,c,d,d +c,,e,d,a,c,c,c,d +b,b,d,b,a,,d,b,c +b,d,e,c,a,c,c,b,c +d,a,f,a,a,,e,a,b +c,a,e,a,a,b,d,c,c +c,c,f,a,a,a,c,a,a +c,d,e,d,a,d,c,c,c +d,a,e,a,a,,c,c,d +a,a,f,a,a,a,e,c,a +d,d,f,b,a,d,c,c,a +c,a,e,c,a,a,d,c,a +d,b,e,d,a,,g,d,c +c,b,e,b,a,d,c,c,b +d,c,e,c,a,b,d,c,c +d,b,f,b,a,b,d,b,b +d,b,e,c,a,b,c,d,c +d,c,e,d,a,b,c,d,c +b,a,e,a,a,d,d,c,d +d,d,e,b,a,a,d,b,b +c,,f,c,a,d,c,b,c +d,d,e,d,a,a,c,c,d +a,d,e,c,a,,c,b,d +a,b,f,a,a,,d,a,a +c,b,d,c,a,c,d,b,a +c,a,e,c,a,,d,b,b +c,d,g,c,a,b,d,d,a +b,c,f,d,a,b,d,b,b +c,c,e,a,a,c,d,d,c +b,c,e,d,a,a,f,d,a +b,a,f,a,a,,f,b,a +b,a,f,c,a,,d,a,b +b,d,e,a,a,b,d,c,a +a,a,e,c,a,a,b,c,b +d,,e,d,a,d,e,d,c +a,b,f,b,a,,d,b,a +c,c,e,d,a,b,b,c,b +d,d,e,b,a,b,d,c,d +a,a,e,a,a,,d,a,a +a,a,e,d,a,,d,d,b +b,,f,d,a,b,d,c,a +c,d,e,c,a,c,c,c,d +d,c,e,c,a,c,f,d,b +b,c,e,b,a,b,f,b,a +b,a,f,b,a,,d,a,a +a,d,h,b,a,,d,a,a +a,d,e,d,a,d,b,b,d +b,d,e,c,a,b,d,b,d +b,c,e,a,a,,c,b,a +d,d,e,d,a,d,c,d,d +b,b,h,b,a,,d,a,a +c,d,e,c,a,b,d,a,d +c,b,e,b,a,,d,d,b +a,b,d,a,a,d,b,d,d +d,b,e,c,a,c,d,d,b +b,a,f,b,a,a,d,c,b +a,d,f,b,a,b,c,d,a +a,a,e,b,a,d,c,c,d +c,b,e,b,a,a,d,b,c +b,b,f,b,a,b,d,b,a +c,c,e,c,a,a,d,a,a +b,c,e,d,a,b,e,d,c +a,a,d,a,a,,d,a,a +c,c,e,b,a,b,c,c,c +d,c,e,d,a,d,d,d,d +c,d,e,c,a,d,c,d,d +c,d,e,c,a,c,f,d,a +c,b,e,c,a,c,d,c,d +c,c,e,b,a,b,d,c,a +c,a,e,d,a,b,d,b,d +d,d,e,d,a,d,b,d,d +d,d,e,d,a,d,c,c,d +b,c,f,a,a,,d,b,b +d,c,f,d,a,d,d,c,d +a,c,f,b,a,,d,b,a +c,d,e,c,a,d,d,b,d +c,a,e,b,a,b,d,c,b +c,,e,c,a,c,c,d,d +c,c,f,a,a,b,d,a,c +b,,f,d,a,b,d,c,b +b,b,e,c,a,c,e,c,b +c,d,d,c,a,c,c,a,d +d,,f,c,a,c,d,c,b +c,,f,c,a,,d,b,c +c,c,e,d,a,c,c,d,c +d,d,e,d,a,a,d,d,a +b,,e,c,a,b,d,b,b +d,d,e,d,a,d,c,b,d +b,c,f,a,a,,e,c,b +b,b,d,b,a,b,d,b,c +d,d,e,d,a,,b,b,c +b,c,e,a,a,a,d,a,a +c,d,e,d,a,c,b,a,c +b,b,f,c,a,,e,c,a +c,c,e,c,a,,d,c,c +c,b,f,c,a,b,c,c,b +a,a,e,d,a,b,c,a,a +d,d,e,d,a,b,c,d,b +d,b,e,d,a,c,e,a,d +d,c,e,c,a,b,c,b,d +b,d,f,a,a,a,e,a,d +b,d,f,d,a,,d,d,c +d,d,i,d,a,d,b,d,d +d,d,e,d,a,a,c,d,c +c,c,f,c,a,a,c,a,d +b,a,f,a,a,,d,d,a +a,b,f,a,a,,c,a,a +b,a,f,a,a,,d,c,b +b,d,f,c,a,,c,b,b +c,b,e,d,a,d,c,d,c +a,d,e,a,a,b,d,d,a +a,a,f,a,a,,d,a,a +c,a,e,b,a,a,d,d,c +d,d,e,d,a,d,c,a,c +b,b,e,b,a,a,e,c,c +b,b,f,a,a,,c,b,a +a,b,e,d,a,c,c,a,a +d,c,e,d,a,a,c,d,d +a,d,e,d,a,b,d,c,c +d,d,e,b,a,c,d,c,b +c,c,e,d,a,d,c,d,d +d,a,e,c,a,,c,a,b +c,c,i,b,a,d,d,b,c +c,,e,a,a,d,e,c,d +d,c,e,d,a,d,f,d,d +c,a,e,c,a,b,c,b,b +b,d,i,a,a,,e,a,a +b,b,e,c,a,b,c,c,b +c,b,e,d,a,,c,c,c +a,c,e,b,a,,c,b,a +c,,e,d,a,b,c,c,b +b,b,e,b,a,b,g,d,d +b,,e,c,a,a,d,c,b +d,c,g,d,a,c,c,b,d +a,a,e,b,a,c,f,d,d +d,a,e,b,a,d,c,d,b +a,a,e,a,a,a,c,a,b +a,a,e,c,a,b,f,c,a +c,d,e,b,a,,d,a,a +a,a,e,c,a,d,f,a,a +b,d,e,c,a,c,e,a,c +b,c,f,b,a,,d,b,b +c,,f,c,a,b,e,b,a +d,a,g,b,a,a,e,c,b +b,b,f,b,a,,d,b,a +c,d,e,d,a,d,c,c,d +c,d,e,c,a,d,e,c,c +d,b,e,c,a,b,c,d,b +c,d,e,b,a,d,c,d,c +b,,e,d,a,,e,d,b +c,d,e,a,a,b,c,c,d +b,c,f,a,a,a,e,a,b +d,d,e,d,a,,c,d,c +b,a,f,b,a,,f,b,c +b,d,f,a,a,c,d,c,c +c,d,e,c,a,d,c,c,d +d,b,h,d,a,a,c,a,b +c,c,f,b,a,c,b,c,b +d,b,e,c,a,a,d,a,c +c,c,g,b,a,d,b,c,d +c,c,d,c,a,,d,c,b +b,d,d,a,a,b,e,b,a +c,c,e,c,a,b,c,c,d +a,,e,b,a,a,d,a,a +b,a,f,b,a,a,d,d,b +c,a,e,c,a,,b,b,a +b,,e,c,a,,e,c,b +c,d,e,c,a,c,d,b,b +c,c,f,c,a,,c,d,a +b,a,e,c,a,c,b,d,a +b,c,f,b,a,b,d,d,a +a,c,e,b,a,a,d,b,a +d,b,e,b,a,b,c,c,b +d,d,e,a,a,b,c,b,a +c,,e,c,a,b,d,c,a +d,d,e,c,a,,c,b,d +d,d,e,d,a,c,c,c,d +d,d,e,d,a,b,c,d,d +c,c,e,d,a,d,d,c,d +d,d,e,c,a,d,c,d,d +d,c,g,d,a,d,c,d,c +c,a,e,a,a,b,e,d,b +d,c,e,b,a,,d,a,a +d,b,e,b,a,a,d,b,a +d,b,e,d,a,d,d,d,d +c,c,e,d,a,b,c,c,d +b,a,f,a,a,a,d,b,a +c,c,e,d,a,b,c,d,a +c,c,f,c,a,b,d,b,b +a,,e,c,a,a,d,c,a +d,a,d,c,a,a,d,a,a +a,d,h,b,a,,e,b,a +c,d,e,d,a,,c,a,c +b,a,f,a,a,,e,a,b +d,a,e,b,a,d,d,b,b +a,c,d,a,a,,c,c,d +d,c,e,c,a,d,c,d,d +c,a,e,c,a,a,d,b,a +b,c,h,d,a,d,d,d,c +a,a,f,a,a,,d,b,a +a,a,e,b,a,,d,b,b +b,b,f,a,a,a,d,b,d +a,b,f,b,a,a,c,a,a +a,d,f,d,a,,c,a,c +b,c,f,b,a,a,d,c,a +c,a,e,b,a,d,e,c,b +a,a,e,a,a,b,c,d,d +b,a,g,d,a,d,d,b,a +c,b,f,d,a,c,b,b,c +c,,e,c,a,c,d,c,c +b,a,f,a,a,c,d,a,a +b,d,e,b,a,a,d,b,d +c,b,e,b,a,,c,a,b +b,c,e,c,a,a,e,a,c +b,b,f,b,a,,d,b,a +d,d,e,d,a,a,c,c,d +d,c,e,a,a,b,d,c,d +a,b,f,a,a,,d,b,a +a,a,e,a,a,a,e,b,a +d,d,e,d,a,c,c,c,d +c,,d,a,a,c,d,c,d +c,c,f,c,a,b,d,d,d +b,c,e,b,a,a,d,a,a +d,d,e,d,a,b,e,c,d +b,,e,d,a,b,c,c,d +a,b,e,b,a,a,f,c,a +b,b,e,a,a,,c,a,b +b,b,f,b,a,b,d,a,a +b,b,e,a,a,b,d,a,b +a,c,f,b,a,,d,b,b +a,c,e,b,a,b,g,c,d +a,a,f,a,a,,d,a,a +a,,e,d,a,a,d,b,a +d,c,e,b,a,,c,c,a +c,d,e,c,a,b,e,b,b +b,d,e,d,a,a,d,a,d +c,d,e,a,a,c,c,c,c +d,b,f,d,a,,c,d,b +c,c,f,b,a,,d,c,c +d,b,e,d,a,b,e,d,b +c,c,e,c,a,d,c,a,c +c,c,e,c,a,b,d,d,d +b,b,f,a,a,b,c,b,c +b,b,e,b,a,a,d,c,b +d,d,e,b,a,c,c,b,b +c,c,e,a,a,c,d,a,d +c,b,e,c,a,a,c,b,c +b,b,f,c,a,,c,a,a +d,b,f,c,a,d,c,c,c +b,b,f,b,a,a,d,b,b +b,c,e,c,a,d,d,d,c +c,c,e,d,a,b,b,b,a +d,,e,b,a,d,c,d,d +d,c,e,b,a,b,d,c,a +b,a,f,a,a,,d,a,b +b,c,e,a,a,,e,c,d +c,b,e,d,a,b,d,c,d +d,c,e,b,a,d,c,d,b +c,a,e,d,a,b,d,c,b +c,c,e,c,a,a,c,b,d +d,a,g,c,a,b,d,b,c +d,a,f,d,a,a,e,d,c +d,a,e,d,a,a,e,d,c +a,c,f,c,a,c,d,b,b +c,c,e,b,a,d,d,d,d +b,c,d,c,a,a,d,a,c +d,c,e,d,a,b,c,d,d +c,,g,c,a,b,d,a,d +d,a,e,d,a,c,d,d,d +b,a,f,a,a,,d,a,a +a,b,e,c,a,a,c,d,a +c,,e,b,a,b,e,c,c +b,b,f,a,a,d,c,b,b +b,b,f,a,a,a,e,d,b +d,,e,d,a,a,d,c,d +d,b,e,b,a,d,e,b,c +c,d,e,d,a,d,d,c,b +c,b,e,c,a,,d,a,b +b,c,e,b,a,,d,b,b +b,d,e,d,a,c,b,a,b +a,c,f,b,a,b,f,a,c +c,,g,b,a,a,d,d,d +d,c,e,d,a,,c,d,b +c,c,e,b,a,b,d,b,d +b,d,e,d,a,d,b,c,d +d,c,e,d,a,c,c,c,d +d,d,e,d,a,b,c,c,a +d,c,e,c,a,d,c,b,b +d,a,f,b,a,c,d,b,c +b,b,e,c,a,c,d,b,b +b,d,f,b,a,,c,c,a +c,c,e,a,a,b,d,a,d +c,c,f,c,a,,d,c,a +a,c,g,c,a,a,d,d,d +b,d,f,a,a,c,d,d,c +a,a,f,a,a,,d,a,a +a,b,e,c,a,b,d,a,a +a,a,e,a,a,d,b,a,b +d,c,f,b,a,,f,b,d +b,c,g,a,a,b,d,c,a +b,,e,b,a,b,c,d,a +c,d,e,c,a,c,c,c,d +b,a,e,d,a,a,d,a,a +b,b,e,b,a,b,c,b,d +d,d,e,d,a,c,d,d,d +d,c,e,d,a,c,c,d,d +b,d,e,b,a,c,d,c,c +b,b,e,b,a,,c,b,a +d,c,e,d,a,d,d,c,b +c,c,e,b,a,c,c,b,d +d,b,e,a,a,d,c,b,c +c,a,f,c,a,a,d,c,a +c,c,f,b,a,d,e,c,b +c,c,e,d,a,d,d,d,d +d,d,e,d,a,,c,d,d +b,,e,a,a,c,c,d,d +c,c,e,d,a,a,b,d,c +c,b,d,d,a,a,d,d,b +c,b,d,b,a,a,d,b,b +b,b,f,d,a,d,d,b,b +d,d,e,d,a,c,d,b,b +b,a,f,a,a,,e,b,a +a,c,e,c,a,a,e,a,a +d,a,e,c,a,a,d,d,d +b,a,f,a,a,a,e,a,b +a,b,f,b,a,a,e,c,b +c,a,f,a,a,d,d,a,a +c,d,e,b,a,,c,d,a +d,a,f,a,a,,d,c,b +b,a,f,b,a,a,c,d,a +d,c,f,c,a,c,f,c,a +a,b,e,a,a,,c,b,d +c,c,f,c,a,a,f,c,d +b,,f,c,a,c,d,b,d +c,c,e,d,a,c,c,d,b +c,d,e,b,a,a,c,d,a +b,a,e,b,a,,e,b,a +c,a,e,c,a,,c,b,b +c,a,f,b,a,,d,a,b +c,b,e,d,a,c,c,c,a +c,,e,a,a,b,d,c,d +c,c,f,c,a,b,c,b,b +a,b,e,b,a,,e,a,a +d,b,e,a,a,,c,b,a +d,c,e,c,a,d,d,a,b +b,d,e,d,a,,g,c,b +c,d,f,c,a,d,b,b,d +c,c,f,a,a,a,d,c,b +d,d,e,b,a,b,b,c,d +c,d,e,a,a,a,d,c,a +b,a,f,a,a,,d,b,a +c,d,e,d,a,d,d,d,d +c,d,e,c,a,b,d,c,c +d,d,d,b,a,,f,b,d +c,c,f,b,a,,b,a,a +b,d,f,a,a,,c,b,a +c,a,e,d,a,b,d,c,c +c,b,e,b,a,d,c,b,d +a,b,e,d,a,c,e,c,b +d,c,f,b,a,,e,d,d +b,b,f,b,a,b,d,c,a +b,c,g,b,a,b,d,a,c +a,,e,d,a,,e,a,a +a,a,h,a,a,c,f,b,a +b,,f,d,a,c,d,c,a +c,c,f,b,a,c,c,c,a +d,d,e,b,a,b,d,c,b +c,d,e,b,a,b,d,d,c +d,c,e,c,a,b,d,b,d +c,c,e,a,a,,c,b,b +a,d,f,b,a,,d,b,a +d,d,e,c,a,c,d,c,d +d,c,f,d,a,b,c,d,b +c,c,g,a,a,c,e,a,d +a,a,f,a,a,c,d,a,b +a,a,e,a,a,c,c,a,d +d,d,e,d,a,d,d,c,a +a,c,f,c,a,,d,b,d +a,c,e,c,a,a,f,b,c +a,d,e,a,a,,f,d,a +b,c,e,c,a,a,c,c,b +a,d,d,b,a,,e,b,b +a,a,f,a,a,,e,a,c +c,d,f,b,a,c,c,b,a +d,d,e,d,a,d,c,c,c +c,a,e,d,a,c,f,b,d +c,b,f,a,a,,d,d,a +c,d,e,c,a,b,b,d,a +c,a,e,c,a,,d,c,b +a,a,d,a,a,c,d,a,a diff --git a/dvc.lock b/dvc.lock index b25d046..dbcbec9 100644 --- a/dvc.lock +++ b/dvc.lock @@ -1,13 +1,13 @@ schema: '2.0' stages: subsample: - cmd: "mkdir -p data/test && python scripts/subsample.py --data data/data.csv --test-data-dir\ - \ data/test --params params.yaml --output data/subsampled.csv\n" + cmd: "mkdir -p data/test && python scripts/subsample.py --data data/data.csv --test-data-dir + data/test --params params.yaml --output data/subsampled.csv\n" deps: - path: data/data.csv hash: md5 - md5: 2652c974be9aa49bd50061377d39563d - size: 313452 + md5: 61a5fd7f3037d01741bff5fd67feed61 + size: 46028 - path: scripts/subsample.py hash: md5 md5: e984c015340dc2e50d926b4e3931b9dd @@ -20,190 +20,201 @@ stages: outs: - path: data/subsampled.csv hash: md5 - md5: 54da26f21f0fee6db642e1761ebe69b9 - size: 268196 + md5: 90bbf6bd98dd82bff9d3da618fe51cc9 + size: 23605 - path: data/test hash: md5 - md5: b8d3099ce11e7976c280f51cf51cbb96.dir - size: 45938 - nfiles: 2 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 validate-column-names: cmd: bin/validate.sh deps: - path: data/subsampled.csv hash: md5 - md5: 54da26f21f0fee6db642e1761ebe69b9 - size: 268196 + md5: 90bbf6bd98dd82bff9d3da618fe51cc9 + size: 23605 outs: - path: data/validated.csv hash: md5 - md5: 54da26f21f0fee6db642e1761ebe69b9 - size: 268196 + md5: 90bbf6bd98dd82bff9d3da618fe51cc9 + size: 23605 nullify: - cmd: "clojure -X gensql.structure-learning.main/nullify < data/validated.csv >\ - \ data/nullified.csv\n" + cmd: "clojure -X gensql.structure-learning.main/nullify < data/validated.csv > + data/nullified.csv\n" deps: - path: data/validated.csv hash: md5 - md5: 54da26f21f0fee6db642e1761ebe69b9 - size: 268196 + md5: 90bbf6bd98dd82bff9d3da618fe51cc9 + size: 23605 params: params.yaml: nullify: + - NaN + - missing outs: - path: data/nullified.csv hash: md5 - md5: 54da26f21f0fee6db642e1761ebe69b9 - size: 268196 + md5: 90bbf6bd98dd82bff9d3da618fe51cc9 + size: 23605 guess-schema: - cmd: "clojure -X gensql.structure-learning.main/guess-schema < data/nullified.csv\ - \ > data/schema.edn\n" + cmd: "clojure -X gensql.structure-learning.main/guess-schema < data/nullified.csv + > data/schema.edn\n" deps: - path: data/nullified.csv hash: md5 - md5: 54da26f21f0fee6db642e1761ebe69b9 - size: 268196 + md5: 90bbf6bd98dd82bff9d3da618fe51cc9 + size: 23605 params: params.yaml: schema: + Municipality: ignore + Total_population__Both_sexes_: nominal + Taxable_income__million_yen_: nominal + Ordinal_Number_of_employed_persons: nominal + Number_of_households__Total_: nominal + Ordinal_Number_of_physicians: nominal + Number_of_dwellings_with_occupying_households: nominal + Ordinal_Total_population__65_and_over_: nominal + Number_of_elderly_nursing_facilities__Questionnaire_about_basic_matters_: nominal + Number_of_lower_secondary_school_students: nominal outs: - path: data/schema.edn hash: md5 - md5: 50689cad3539b17cfc255d6dc2545e1c - size: 601 + md5: 5520eb91d4bc6d08a6ee45d7c4a19a74 + size: 485 cgpm-schema: - cmd: "clojure -X gensql.structure-learning.main/cgpm-schema < data/schema.edn\ - \ > data/cgpm-schema.edn\n" + cmd: "clojure -X gensql.structure-learning.main/cgpm-schema < data/schema.edn + > data/cgpm-schema.edn\n" deps: - path: data/schema.edn hash: md5 - md5: 50689cad3539b17cfc255d6dc2545e1c - size: 601 + md5: 5520eb91d4bc6d08a6ee45d7c4a19a74 + size: 485 outs: - path: data/cgpm-schema.edn hash: md5 - md5: 7e6b6810bece6d68677b7f69b0a9bb6b - size: 186 + md5: 760794aebce88e08dd14596b46311ebe + size: 505 ignore: cmd: "clojure -X gensql.structure-learning.main/ignore :schema '\"data/schema.edn\"\ ' < data/nullified.csv > data/ignored.csv\n" deps: - path: data/nullified.csv hash: md5 - md5: 54da26f21f0fee6db642e1761ebe69b9 - size: 268196 + md5: 90bbf6bd98dd82bff9d3da618fe51cc9 + size: 23605 - path: data/schema.edn hash: md5 - md5: 50689cad3539b17cfc255d6dc2545e1c - size: 601 + md5: 5520eb91d4bc6d08a6ee45d7c4a19a74 + size: 485 outs: - path: data/ignored.csv hash: md5 - md5: 5ffeb41f4182b8f7f062b5f13230557d - size: 77540 + md5: 0f31ce8a5706de864b2c503a540b4ac1 + size: 17911 numericalize: cmd: "clojure -X gensql.structure-learning.main/numericalize :schema '\"data/schema.edn\"\ ' :table data/mapping-table.edn < data/ignored.csv > data/numericalized.csv\n" deps: - path: data/ignored.csv hash: md5 - md5: 5ffeb41f4182b8f7f062b5f13230557d - size: 77540 + md5: 0f31ce8a5706de864b2c503a540b4ac1 + size: 17911 - path: data/schema.edn hash: md5 - md5: 50689cad3539b17cfc255d6dc2545e1c - size: 601 + md5: 5520eb91d4bc6d08a6ee45d7c4a19a74 + size: 485 outs: - path: data/mapping-table.edn hash: md5 - md5: 1d3e7314f20a2b2d50deff70b323f428 - size: 3210 + md5: d5fea59455202ff2741e813364e0c1e8 + size: 647 - path: data/numericalized.csv hash: md5 - md5: 345eaad4a50da45d1f4d50d326de0f2b - size: 11932 + md5: c3f1c52ae42af0e4e18afa96bec9834d + size: 17911 loom-schema: - cmd: "clojure -X gensql.structure-learning.main/loom-schema < data/schema.edn\ - \ > data/loom-schema.json\n" + cmd: "clojure -X gensql.structure-learning.main/loom-schema < data/schema.edn + > data/loom-schema.json\n" deps: - path: data/schema.edn hash: md5 - md5: 50689cad3539b17cfc255d6dc2545e1c - size: 601 + md5: 5520eb91d4bc6d08a6ee45d7c4a19a74 + size: 485 outs: - path: data/loom-schema.json hash: md5 - md5: 68f027a12982e5d4f01f40fe0450b281 - size: 127 + md5: c7f8e3dcb424006ec9aaf56ddf23de3b + size: 416 loom-ingest: cmd: ./bin/loom_task ingest loom data/loom-schema.json data/numericalized.csv deps: - path: data/loom-schema.json hash: md5 - md5: 68f027a12982e5d4f01f40fe0450b281 - size: 127 + md5: c7f8e3dcb424006ec9aaf56ddf23de3b + size: 416 - path: data/numericalized.csv hash: md5 - md5: 345eaad4a50da45d1f4d50d326de0f2b - size: 11932 + md5: c3f1c52ae42af0e4e18afa96bec9834d + size: 17911 outs: - path: loom/ingest hash: md5 - md5: 442c82a1ca799f1a0a92ec930bf6fa9b.dir - size: 17782 + md5: d9bd017d6752b3991f1e41fdcf15d3ec.dir + size: 16257 nfiles: 7 - path: loom/query hash: md5 - md5: 6e16965d6127bf5d755ebfb8b4f558c2.dir + md5: fb1f0540b724515dda7b04fc59a49875.dir size: 106 nfiles: 1 loom-infer-config: cmd: "clojure -X gensql.structure-learning.main/infer-config < params.yaml > data/infer-config.json\n" params: params.yaml: - loom.extra_passes: 1 + loom.extra_passes: 100 seed: 0 outs: - path: data/infer-config.json hash: md5 - md5: 3781d3c543c1a1f8977a7d2ff1c4601c - size: 40 + md5: b9fe21c5095ee00c8fb4a6bb0f002cd1 + size: 42 loom-infer: - cmd: ./bin/loom_task infer loom 3 data/infer-config.json + cmd: ./bin/loom_task infer loom 1 data/infer-config.json deps: - path: data/infer-config.json hash: md5 - md5: 3781d3c543c1a1f8977a7d2ff1c4601c - size: 40 + md5: b9fe21c5095ee00c8fb4a6bb0f002cd1 + size: 42 - path: loom/ingest hash: md5 - md5: 442c82a1ca799f1a0a92ec930bf6fa9b.dir - size: 17782 + md5: d9bd017d6752b3991f1e41fdcf15d3ec.dir + size: 16257 nfiles: 7 - path: loom/query hash: md5 - md5: 6e16965d6127bf5d755ebfb8b4f558c2.dir + md5: fb1f0540b724515dda7b04fc59a49875.dir size: 106 nfiles: 1 params: params.yaml: - sample_count: 3 + sample_count: 1 outs: - path: loom/samples hash: md5 - md5: da9a3174280400cb1fd369db15678dd7.dir - size: 52543 - nfiles: 22 + md5: 840b7af41958ba0b1737b45d58587c5f.dir + size: 103719 + nfiles: 7 loom-dump-metadata: - cmd: "mkdir -p data/cgpm/raw && find loom/samples -mindepth 1 -maxdepth 1 -type\ - \ d | parallel --verbose ./bin/loom python scripts/loom_dump.py {} --output\ - \ data/cgpm/raw/{/}.json\n" + cmd: "mkdir -p data/cgpm/raw && find loom/samples -mindepth 1 -maxdepth 1 -type + d | parallel --verbose ./bin/loom python scripts/loom_dump.py {} --output data/cgpm/raw/{/}.json\n" deps: - path: loom/samples hash: md5 - md5: da9a3174280400cb1fd369db15678dd7.dir - size: 52543 - nfiles: 22 + md5: 840b7af41958ba0b1737b45d58587c5f.dir + size: 103719 + nfiles: 7 - path: scripts/loom_dump.py hash: md5 md5: c43616fe12258e9314fe1b481a6971e8 @@ -214,34 +225,33 @@ stages: outs: - path: data/cgpm/raw hash: md5 - md5: e575cb2e33443ba82f8d4c450180a5bc.dir - size: 12284 - nfiles: 3 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 cgpm-hydrate-metadata: - cmd: "find data/cgpm/raw -type f | parallel jsonschema --instance {} schemas/cgpm.json\ - \ && mkdir -p data/cgpm/hydrated && rm -f data/cgpm/inf.log && find data/cgpm/raw\ - \ -type f | sort | parallel --verbose 'python scripts/cgpm_hydrate.py --metadata\ - \ {} --output data/cgpm/hydrated/{/} --data data/numericalized.csv --schema\ - \ data/cgpm-schema.edn --mapping-table data/mapping-table.edn --seed $((0 +\ - \ {#} - 1))'\n" + cmd: "find data/cgpm/raw -type f | parallel jsonschema --instance {} schemas/cgpm.json + && mkdir -p data/cgpm/hydrated && rm -f data/cgpm/inf.log && find data/cgpm/raw + -type f | sort | parallel --verbose 'python scripts/cgpm_hydrate.py --metadata + {} --output data/cgpm/hydrated/{/} --data data/numericalized.csv --schema data/cgpm-schema.edn + --mapping-table data/mapping-table.edn --seed $((0 + {#} - 1))'\n" deps: - path: data/cgpm-schema.edn hash: md5 - md5: 7e6b6810bece6d68677b7f69b0a9bb6b - size: 186 + md5: 760794aebce88e08dd14596b46311ebe + size: 505 - path: data/cgpm/raw hash: md5 - md5: e575cb2e33443ba82f8d4c450180a5bc.dir - size: 12284 - nfiles: 3 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 - path: data/mapping-table.edn hash: md5 - md5: 1d3e7314f20a2b2d50deff70b323f428 - size: 3210 + md5: d5fea59455202ff2741e813364e0c1e8 + size: 647 - path: data/numericalized.csv hash: md5 - md5: 345eaad4a50da45d1f4d50d326de0f2b - size: 11932 + md5: c3f1c52ae42af0e4e18afa96bec9834d + size: 17911 - path: schemas/cgpm.json hash: md5 md5: 75fb21013e98a73e03fbe1ace14426f2 @@ -257,26 +267,26 @@ stages: outs: - path: data/cgpm/hydrated hash: md5 - md5: f0b203a84011a47c6efc7ce8e411898e.dir - size: 125827 - nfiles: 3 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 cgpm-infer-hyperparameters: - cmd: "find data/cgpm/hydrated -type f | parallel jsonschema --instance {} schemas/cgpm.json\ - \ && mkdir -p data/cgpm/complete && echo 1 >> data/cgpm/inf.log && find data/cgpm/hydrated\ - \ -type f | sort | parallel --verbose 'python scripts/cgpm_infer.py {} --kernel\ - \ alpha --kernel view_alphas --kernel column_hypers --kernel rows --kernel columns\ - \ --output data/cgpm/complete/{/} --data data/numericalized.csv --params params.yaml\ - \ --seed $((0 + {#} - 1)) --minutes 1' #--iterations 1\n" + cmd: "find data/cgpm/hydrated -type f | parallel jsonschema --instance {} schemas/cgpm.json + && mkdir -p data/cgpm/complete && echo 450 >> data/cgpm/inf.log && find data/cgpm/hydrated + -type f | sort | parallel --verbose 'python scripts/cgpm_infer.py {} --kernel + alpha --kernel view_alphas --kernel column_hypers --kernel rows --kernel columns + --output data/cgpm/complete/{/} --data data/numericalized.csv --params params.yaml + --seed $((0 + {#} - 1)) --minutes 450' #--iterations 1\n" deps: - path: data/cgpm/hydrated hash: md5 - md5: f0b203a84011a47c6efc7ce8e411898e.dir - size: 125827 - nfiles: 3 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 - path: data/numericalized.csv hash: md5 - md5: 345eaad4a50da45d1f4d50d326de0f2b - size: 11932 + md5: c3f1c52ae42af0e4e18afa96bec9834d + size: 17911 - path: schemas/cgpm.json hash: md5 md5: 75fb21013e98a73e03fbe1ace14426f2 @@ -289,38 +299,38 @@ stages: params.yaml: cgpm: iterations: 1 - minutes: 1 + minutes: 450 parallel.flags: --verbose seed: 0 outs: - path: data/cgpm/complete hash: md5 - md5: b01aaee70e74d3ad3b3dcca27ab61313.dir - size: 117849 - nfiles: 3 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 save-dependencies: - cmd: "find data/cgpm/complete -type f | sort | xargs python scripts/dep_prob.py\ - \ --data data/numericalized.csv --output data/dep-prob.json\n" + cmd: "find data/cgpm/complete -type f | sort | xargs python scripts/dep_prob.py + --data data/numericalized.csv --output data/dep-prob.json\n" deps: - path: data/cgpm/complete hash: md5 - md5: b01aaee70e74d3ad3b3dcca27ab61313.dir - size: 117849 - nfiles: 3 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 outs: - path: data/dep-prob.json hash: md5 - md5: 03184adcfc0c6721bd5cdabef8098169 - size: 774 + md5: 9f23e3e3b128e86caca2a74ee57128b4 + size: 3789 save-max-number-views: - cmd: "find data/cgpm/complete -type f | xargs python scripts/save_n_views.py >>\ - \ data/max-number-views.txt\n" + cmd: "find data/cgpm/complete -type f | xargs python scripts/save_n_views.py >> + data/max-number-views.txt\n" deps: - path: data/cgpm/complete hash: md5 - md5: b01aaee70e74d3ad3b3dcca27ab61313.dir - size: 117849 - nfiles: 3 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 outs: - path: data/max-number-views.txt hash: md5 @@ -328,13 +338,13 @@ stages: size: 1 dep-prob-vl: cmd: "clojure -X gensql.structure-learning.heatmap/vega-lite :stats-path '\"data/dep-prob.json\"\ - ' :domain '[0.0 1.0]' :default 1.0 :name '\"crosscat\"' :scheme '\"blues\"'\ - \ > data/dep-prob.vl.json\n" + ' :domain '[0.0 1.0]' :default 1.0 :name '\"crosscat\"' :scheme '\"blues\"' + > data/dep-prob.vl.json\n" deps: - path: data/dep-prob.json hash: md5 - md5: 03184adcfc0c6721bd5cdabef8098169 - size: 774 + md5: 9f23e3e3b128e86caca2a74ee57128b4 + size: 3789 - path: src/clojure/gensql/structure_learning/heatmap.clj hash: md5 md5: 1a9cd1bb8a534431a3836ed066758616 @@ -342,44 +352,44 @@ stages: outs: - path: data/dep-prob.vl.json hash: md5 - md5: e8678ddc010f3b17c3d3d7ede9d09679 - size: 2852 + md5: c0ffdb4672a6367359bfa6f4d5028eb0 + size: 10303 dep-prob-vg: cmd: "pnpm vl2vg < data/dep-prob.vl.json > data/dep-prob.vg.json\n" deps: - path: data/dep-prob.vl.json hash: md5 - md5: e8678ddc010f3b17c3d3d7ede9d09679 - size: 2852 + md5: c0ffdb4672a6367359bfa6f4d5028eb0 + size: 10303 outs: - path: data/dep-prob.vg.json hash: md5 - md5: 90b6c1771e1d158da81074ad448005d8 - size: 4675 + md5: ebf7dd5ffd0cf3cf502eac3b622978a8 + size: 12192 dep-prob-svg: cmd: "pnpm vg2svg < data/dep-prob.vg.json > data/dep-prob.svg\n" deps: - path: data/dep-prob.vg.json hash: md5 - md5: 90b6c1771e1d158da81074ad448005d8 - size: 4675 + md5: ebf7dd5ffd0cf3cf502eac3b622978a8 + size: 12192 outs: - path: data/dep-prob.svg hash: md5 - md5: 0718cc6f9322663ea01449cf858ff65b - size: 13898 + md5: 6470c695c5e27880a069417397d9f6a0 + size: 28176 save-linear-stats: - cmd: "python scripts/linear_stats.py --data data/ignored.csv --schema data/schema.edn\ - \ --output data/linear-stats.json\n" + cmd: "python scripts/linear_stats.py --data data/ignored.csv --schema data/schema.edn + --output data/linear-stats.json\n" deps: - path: data/ignored.csv hash: md5 - md5: 5ffeb41f4182b8f7f062b5f13230557d - size: 77540 + md5: 0f31ce8a5706de864b2c503a540b4ac1 + size: 17911 - path: data/schema.edn hash: md5 - md5: 50689cad3539b17cfc255d6dc2545e1c - size: 601 + md5: 5520eb91d4bc6d08a6ee45d7c4a19a74 + size: 485 - path: scripts/linear_stats.py hash: md5 md5: d5ebd4d1195a949b7c66ea7b77447f11 @@ -387,21 +397,21 @@ stages: outs: - path: data/linear-stats.json hash: md5 - md5: d5e5ecd37ebd8d5cd03c1bbd6a6fe7f6 - size: 2899 + md5: 6dde33046d38296b9adb8a8fd995b347 + size: 8056 linear-stats-vl: cmd: "clojure -X gensql.structure-learning.heatmap/vega-lite :stats-path '\"data/linear-stats.json\"\ - ' :sort-path '\"data/dep-prob.json\"' :domain '[1.0 0.0]' :default 0.0 :name\ - \ '\"statistics\"' :field '\"p-value\"' :scheme '\"oranges\"' > data/linear-stats.vl.json\n" + ' :sort-path '\"data/dep-prob.json\"' :domain '[1.0 0.0]' :default 0.0 :name + '\"statistics\"' :field '\"p-value\"' :scheme '\"oranges\"' > data/linear-stats.vl.json\n" deps: - path: data/dep-prob.json hash: md5 - md5: 03184adcfc0c6721bd5cdabef8098169 - size: 774 + md5: 9f23e3e3b128e86caca2a74ee57128b4 + size: 3789 - path: data/linear-stats.json hash: md5 - md5: d5e5ecd37ebd8d5cd03c1bbd6a6fe7f6 - size: 2899 + md5: 6dde33046d38296b9adb8a8fd995b347 + size: 8056 - path: src/clojure/gensql/structure_learning/heatmap.clj hash: md5 md5: 1a9cd1bb8a534431a3836ed066758616 @@ -409,67 +419,67 @@ stages: outs: - path: data/linear-stats.vl.json hash: md5 - md5: 1d564d040268106c284b268dd7618abd - size: 3489 + md5: 3af41ba4c8ba4e17dfd3a5d332ecd271 + size: 11304 linear-stats-vg: cmd: "pnpm vl2vg < data/linear-stats.vl.json > data/linear-stats.vg.json\n" deps: - path: data/linear-stats.vl.json hash: md5 - md5: 1d564d040268106c284b268dd7618abd - size: 3489 + md5: 3af41ba4c8ba4e17dfd3a5d332ecd271 + size: 11304 outs: - path: data/linear-stats.vg.json hash: md5 - md5: a7e6eba1560d847892d815d127e76d8a - size: 5381 + md5: d92235b318a68b7ca1ba940356b51002 + size: 13290 linear-stats-svg: cmd: "pnpm vg2svg < data/linear-stats.vg.json > data/linear-stats.svg\n" deps: - path: data/linear-stats.vg.json hash: md5 - md5: a7e6eba1560d847892d815d127e76d8a - size: 5381 + md5: d92235b318a68b7ca1ba940356b51002 + size: 13290 outs: - path: data/linear-stats.svg hash: md5 - md5: 7eb988a7222ad2519f5819f6ea22da85 - size: 14507 + md5: f7d051b7034a4447b74be3c44cec29d1 + size: 29104 compare-dep-prob-with-linear: - cmd: "python scripts/compare_deps.py --deps data/dep-prob.json --linear data/linear-stats.json\ - \ >> data/qc-statistical-tests.txt\n" + cmd: "python scripts/compare_deps.py --deps data/dep-prob.json --linear data/linear-stats.json + >> data/qc-statistical-tests.txt\n" deps: - path: data/dep-prob.json hash: md5 - md5: 03184adcfc0c6721bd5cdabef8098169 - size: 774 + md5: 9f23e3e3b128e86caca2a74ee57128b4 + size: 3789 - path: data/linear-stats.json hash: md5 - md5: d5e5ecd37ebd8d5cd03c1bbd6a6fe7f6 - size: 2899 + md5: 6dde33046d38296b9adb8a8fd995b347 + size: 8056 outs: - path: data/qc-statistical-tests.txt hash: md5 - md5: 0d13981f8869abf58feab894ba234aa2 - size: 344 + md5: f71635a88039f2c13051b44b74acc0bb + size: 1049 ast-export: cmd: - 'parallel --verbose jsonschema --instance {} schemas/cgpm.json :::: <(find data/cgpm/complete -type f)' - mkdir -p data/ast - - "parallel --verbose 'python scripts/ast_export.py --metadata {} --data data/numericalized.csv\ - \ --mapping-table data/mapping-table.edn --output data/ast/{/.}.edn' :::: <(find\ - \ data/cgpm/complete -type f)" + - "parallel --verbose 'python scripts/ast_export.py --metadata {} --data data/numericalized.csv + --mapping-table data/mapping-table.edn --output data/ast/{/.}.edn' :::: <(find + data/cgpm/complete -type f)" deps: - path: data/cgpm/complete hash: md5 - md5: b01aaee70e74d3ad3b3dcca27ab61313.dir - size: 117849 - nfiles: 3 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 - path: data/ignored.csv hash: md5 - md5: 5ffeb41f4182b8f7f062b5f13230557d - size: 77540 + md5: 0f31ce8a5706de864b2c503a540b4ac1 + size: 17911 - path: scripts/ast_export.py hash: md5 md5: 301e377deb0aaf75b1a961a6d91153a8 @@ -480,21 +490,21 @@ stages: outs: - path: data/ast hash: md5 - md5: d723a36d25e183298f79243c03aaa292.dir - size: 98691 - nfiles: 3 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 sppl-import: cmd: - mkdir -p data/sppl/unmerged - - "parallel --verbose 'python scripts/sppl_import.py --multi-mix-ast {} --max-number-views\ - \ data/max-number-views.txt --output data/sppl/unmerged/{/.}.json' :::: <(find\ - \ data/ast -type f)" + - "parallel --verbose 'python scripts/sppl_import.py --multi-mix-ast {} --max-number-views + data/max-number-views.txt --output data/sppl/unmerged/{/.}.json' :::: <(find + data/ast -type f)" deps: - path: data/ast hash: md5 - md5: d723a36d25e183298f79243c03aaa292.dir - size: 98691 - nfiles: 3 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 - path: data/max-number-views.txt hash: md5 md5: c4ca4238a0b923820dcc509a6f75849b @@ -509,35 +519,35 @@ stages: outs: - path: data/sppl/unmerged hash: md5 - md5: 857a3be1934d586ff87c54821129a3f2.dir - size: 77713 - nfiles: 3 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 sppl-merge: - cmd: "find data/sppl/unmerged -type f | sort | xargs python scripts/sppl_merge.py\ - \ --output data/sppl/merged.json\n" + cmd: "find data/sppl/unmerged -type f | sort | xargs python scripts/sppl_merge.py + --output data/sppl/merged.json\n" deps: - path: data/sppl/unmerged hash: md5 - md5: 857a3be1934d586ff87c54821129a3f2.dir - size: 77713 - nfiles: 3 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 outs: - path: data/sppl/merged.json hash: md5 - md5: ca230a9375be713e0776d2dd1d24ee58 - size: 78194 + md5: 9b43b5adb620c8592befe130bb97db37 + size: 15784 sppl-sample: - cmd: "python scripts/sppl_sample.py --model data/sppl/merged.json --data data/ignored.csv\ - \ > data/synthetic-data-gensql.csv # --sample_count 1000\n" + cmd: "python scripts/sppl_sample.py --model data/sppl/merged.json --data data/ignored.csv + > data/synthetic-data-gensql.csv # --sample_count 1000\n" deps: - path: data/ignored.csv hash: md5 - md5: 5ffeb41f4182b8f7f062b5f13230557d - size: 77540 + md5: 0f31ce8a5706de864b2c503a540b4ac1 + size: 17911 - path: data/sppl/merged.json hash: md5 - md5: ca230a9375be713e0776d2dd1d24ee58 - size: 78194 + md5: 9b43b5adb620c8592befe130bb97db37 + size: 15784 - path: scripts/sppl_sample.py hash: md5 md5: 65c711874e0eda0e7d79b4a4a3eb6076 @@ -545,20 +555,20 @@ stages: outs: - path: data/synthetic-data-gensql.csv hash: md5 - md5: a46403f9fa02b0dc465eac9015bd45d0 - size: 84636 + md5: c0b97869f77a0366fa8c94af8b9e54ed + size: 18352 qc-tag-samples: - cmd: "clojure -X gensql.structure-learning.qc.samples/tag :data data/ignored.csv\ - \ :samples-synthetic data/synthetic-data-gensql.csv > qc/samples/samples.edn\n" + cmd: "clojure -X gensql.structure-learning.qc.samples/tag :data data/ignored.csv + :samples-synthetic data/synthetic-data-gensql.csv > qc/samples/samples.edn\n" deps: - path: data/ignored.csv hash: md5 - md5: 5ffeb41f4182b8f7f062b5f13230557d - size: 77540 + md5: 0f31ce8a5706de864b2c503a540b4ac1 + size: 17911 - path: data/synthetic-data-gensql.csv hash: md5 - md5: a46403f9fa02b0dc465eac9015bd45d0 - size: 84636 + md5: c0b97869f77a0366fa8c94af8b9e54ed + size: 18352 - path: src/clojure/gensql/structure_learning/qc/samples.clj hash: md5 md5: e34e26db3736bb02dd084800c9d6af72 @@ -566,37 +576,37 @@ stages: outs: - path: qc/samples/samples.edn hash: md5 - md5: 67456841283640cd962e574599a73be9 - size: 442526 + md5: a678c18efbaf5b964085e139750818c8 + size: 882785 xcat-complete-import: cmd: - mkdir -p data/xcat/complete/ - - "parallel --verbose 'clojure -X gensql.structure-learning.xcat/import :cgpm-json\ - \ {} :data-csv data/ignored.csv :mapping-table data/mapping-table.edn :numericalized-csv\ - \ data/numericalized.csv :schema-edn data/schema.edn > data/xcat/complete/{/.}.edn'\ - \ :::: <(find data/cgpm/complete -type f)" + - "parallel --verbose 'clojure -X gensql.structure-learning.xcat/import :cgpm-json + {} :data-csv data/ignored.csv :mapping-table data/mapping-table.edn :numericalized-csv + data/numericalized.csv :schema-edn data/schema.edn > data/xcat/complete/{/.}.edn' + :::: <(find data/cgpm/complete -type f)" deps: - path: data/cgpm/complete hash: md5 - md5: b01aaee70e74d3ad3b3dcca27ab61313.dir - size: 117849 - nfiles: 3 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 - path: data/ignored.csv hash: md5 - md5: 5ffeb41f4182b8f7f062b5f13230557d - size: 77540 + md5: 0f31ce8a5706de864b2c503a540b4ac1 + size: 17911 - path: data/mapping-table.edn hash: md5 - md5: 1d3e7314f20a2b2d50deff70b323f428 - size: 3210 + md5: d5fea59455202ff2741e813364e0c1e8 + size: 647 - path: data/numericalized.csv hash: md5 - md5: 345eaad4a50da45d1f4d50d326de0f2b - size: 11932 + md5: c3f1c52ae42af0e4e18afa96bec9834d + size: 17911 - path: data/schema.edn hash: md5 - md5: 50689cad3539b17cfc255d6dc2545e1c - size: 601 + md5: 5520eb91d4bc6d08a6ee45d7c4a19a74 + size: 485 - path: src/clojure/gensql/structure_learning/xcat.clj hash: md5 md5: 50de15b94242c6c71b6822f11cb58771 @@ -607,39 +617,39 @@ stages: outs: - path: data/xcat/complete/ hash: md5 - md5: 56cb77fd3801d7cbf5e7504284d45003.dir - size: 783240 - nfiles: 3 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 xcat-merge: - cmd: "clj -X gensql.structure-learning.clojurecat/merge :models data/xcat/complete\ - \ :out data/xcat/xcat.merged.edn\n" + cmd: "clj -X gensql.structure-learning.clojurecat/merge :models data/xcat/complete + :out data/xcat/xcat.merged.edn\n" deps: - path: data/xcat/complete hash: md5 - md5: 56cb77fd3801d7cbf5e7504284d45003.dir - size: 783240 - nfiles: 3 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 outs: - path: data/xcat/xcat.merged.edn hash: md5 - md5: fee4f7c7a25a88fecb6a2de9e75ef5b0 - size: 783288 + md5: 6f3da0fd910ecd77f789c7bf9db9edf4 + size: 242028 qc-dashboard-spec: - cmd: "clojure -X gensql.structure-learning.qc.dashboard/spec :samples qc/samples/samples.edn\ - \ :schema data/schema.edn :correlation data/linear-stats.json > qc/specs/qc-dashboard.vl.json\n" + cmd: "clojure -X gensql.structure-learning.qc.dashboard/spec :samples qc/samples/samples.edn + :schema data/schema.edn :correlation data/linear-stats.json > qc/specs/qc-dashboard.vl.json\n" deps: - path: data/linear-stats.json hash: md5 - md5: d5e5ecd37ebd8d5cd03c1bbd6a6fe7f6 - size: 2899 + md5: 6dde33046d38296b9adb8a8fd995b347 + size: 8056 - path: data/schema.edn hash: md5 - md5: 50689cad3539b17cfc255d6dc2545e1c - size: 601 + md5: 5520eb91d4bc6d08a6ee45d7c4a19a74 + size: 485 - path: qc/samples/samples.edn hash: md5 - md5: 67456841283640cd962e574599a73be9 - size: 442526 + md5: a678c18efbaf5b964085e139750818c8 + size: 882785 - path: src/clojure/gensql/structure_learning/qc/dashboard.clj hash: md5 md5: 11529f94587a9ade1d3fa4cb84ff0445 @@ -654,17 +664,17 @@ stages: outs: - path: qc/specs/qc-dashboard.vl.json hash: md5 - md5: 47353536a0e2c56390d20c2fc3892a1d - size: 497826 + md5: db641abaa1e4c2255e4f64a880ff3260 + size: 1016130 qc-dashboard-app: - cmd: "clojure -X gensql.structure-learning.qc.app/create :template qc/templates/index.html\ - \ :spec qc/specs/qc-dashboard.vl.json :renderer canvas :title '\"QC dashboard\"\ + cmd: "clojure -X gensql.structure-learning.qc.app/create :template qc/templates/index.html + :spec qc/specs/qc-dashboard.vl.json :renderer canvas :title '\"QC dashboard\"\ ' > qc/app/qc-dashboard.html\n" deps: - path: qc/specs/qc-dashboard.vl.json hash: md5 - md5: 47353536a0e2c56390d20c2fc3892a1d - size: 497826 + md5: db641abaa1e4c2255e4f64a880ff3260 + size: 1016130 - path: qc/templates/index.html hash: md5 md5: ac51efb107febc1caf14b5e2c6d0489d @@ -676,24 +686,24 @@ stages: outs: - path: qc/app/qc-dashboard.html hash: md5 - md5: e3402d22152c64e955cf3388766534f2 - size: 499500 + md5: 4e53b9d04ed61e5b5eb215f3ba24bf5a + size: 1017804 qc-splom-spec: - cmd: "clojure -X gensql.structure-learning.qc.splom/spec :samples qc/samples/samples.edn\ - \ :schema data/schema.edn :correlation data/linear-stats.json > qc/specs/qc-splom.vl.json\n" + cmd: "clojure -X gensql.structure-learning.qc.splom/spec :samples qc/samples/samples.edn + :schema data/schema.edn :correlation data/linear-stats.json > qc/specs/qc-splom.vl.json\n" deps: - path: data/linear-stats.json hash: md5 - md5: d5e5ecd37ebd8d5cd03c1bbd6a6fe7f6 - size: 2899 + md5: 6dde33046d38296b9adb8a8fd995b347 + size: 8056 - path: data/schema.edn hash: md5 - md5: 50689cad3539b17cfc255d6dc2545e1c - size: 601 + md5: 5520eb91d4bc6d08a6ee45d7c4a19a74 + size: 485 - path: qc/samples/samples.edn hash: md5 - md5: 67456841283640cd962e574599a73be9 - size: 442526 + md5: a678c18efbaf5b964085e139750818c8 + size: 882785 - path: src/clojure/gensql/structure_learning/qc/splom.clj hash: md5 md5: a753879cef043c55d23dea202b41abe5 @@ -708,17 +718,17 @@ stages: outs: - path: qc/specs/qc-splom.vl.json hash: md5 - md5: 171da5a9e3af0f572f79596fff4b5b0b - size: 488635 + md5: e627be748e6230d7b0d7448d48f2b888 + size: 951381 qc-splom-app: - cmd: "clojure -X gensql.structure-learning.qc.app/create :template qc/templates/index.html\ - \ :spec qc/specs/qc-splom.vl.json :renderer canvas :title '\"QC scatter plot\ - \ matrix\"' > qc/app/qc-splom.html\n" + cmd: "clojure -X gensql.structure-learning.qc.app/create :template qc/templates/index.html + :spec qc/specs/qc-splom.vl.json :renderer canvas :title '\"QC scatter plot matrix\"\ + ' > qc/app/qc-splom.html\n" deps: - path: qc/specs/qc-splom.vl.json hash: md5 - md5: 171da5a9e3af0f572f79596fff4b5b0b - size: 488635 + md5: e627be748e6230d7b0d7448d48f2b888 + size: 951381 - path: qc/templates/index.html hash: md5 md5: ac51efb107febc1caf14b5e2c6d0489d @@ -730,29 +740,29 @@ stages: outs: - path: qc/app/qc-splom.html hash: md5 - md5: 02b78a8efb352785b73f50856d79f620 - size: 490319 + md5: 9418568e87a7259dccb9dc70d776ab0b + size: 953065 predict: - cmd: "python scripts/predict.py --schema data/schema.edn --training data/ignored.csv\ - \ --training data/synthetic-data-gensql.csv --test data/test/test.csv --output\ - \ data/predictions.csv #--test data/test/test-shifted.csv\n" + cmd: "python scripts/predict.py --schema data/schema.edn --training data/ignored.csv + --training data/synthetic-data-gensql.csv --test data/test/test.csv --output + data/predictions.csv #--test data/test/test-shifted.csv\n" deps: - path: data/ignored.csv hash: md5 - md5: 5ffeb41f4182b8f7f062b5f13230557d - size: 77540 + md5: 0f31ce8a5706de864b2c503a540b4ac1 + size: 17911 - path: data/schema.edn hash: md5 - md5: 50689cad3539b17cfc255d6dc2545e1c - size: 601 + md5: 5520eb91d4bc6d08a6ee45d7c4a19a74 + size: 485 - path: data/synthetic-data-gensql.csv hash: md5 - md5: a46403f9fa02b0dc465eac9015bd45d0 - size: 84636 + md5: c0b97869f77a0366fa8c94af8b9e54ed + size: 18352 - path: data/test/test.csv hash: md5 - md5: 6bdd1f24302e6ee6c1836bf15edfada2 - size: 45597 + md5: 087b8243ca14989643a6404cc42e5b55 + size: 22788 - path: scripts/predict.py hash: md5 md5: 9791ef81e7424516b5c452380c448706 @@ -763,30 +773,32 @@ stages: predictor: Random_forest datasets: - synthetic-data-gensql.csv + - synthetic-data-gan.csv + - synthetic-data-copula.csv outs: - path: data/predictions.csv hash: md5 - md5: a4a35684fdfdf5da4e501d7f4833026a - size: 24264 + md5: 7efd7f5ac4fdb0ec956c5a38b85c5148 + size: 134269 assemble-database: - cmd: "clojure -M:exec gensql.structure-learning.main assemble-database --table-name\ - \ data --table-path data/nullified.csv --model-name baseline_model --model-path\ - \ data/xcat/xcat.merged.edn > data/db.edn\n" + cmd: "clojure -M:exec gensql.structure-learning.main assemble-database --table-name + data --table-path data/nullified.csv --model-name baseline_model --model-path + data/xcat/xcat.merged.edn > data/db.edn\n" deps: - path: data/cgpm hash: md5 - md5: 166c893fcb41ec48e6955a120eadbd5e.dir - size: 255962 - nfiles: 10 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 - path: data/nullified.csv hash: md5 - md5: 54da26f21f0fee6db642e1761ebe69b9 - size: 268196 + md5: 90bbf6bd98dd82bff9d3da618fe51cc9 + size: 23605 - path: data/xcat hash: md5 - md5: 8986883161dcc85bd1cd9b022fd2a29a.dir - size: 1566528 - nfiles: 4 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 params: params.yaml: database.model_name: baseline_model @@ -796,25 +808,25 @@ stages: outs: - path: data/db.edn hash: md5 - md5: bfc7ea1d137db6900ceff8775f784707 - size: 1422959 + md5: 9c6b4d98137899149d79aefc2bcd7869 + size: 657992 qc-discretize@synthetic-data-gensql.csv: - cmd: "mkdir -p data/discretized/ && python scripts/discretize.py --real data/ignored.csv\ - \ --synthetic data/synthetic-data-gensql.csv --schema data/loom-schema.json\ - \ --real-disc data/discretized/ignored.csv --synthetic-disc data/discretized/synthetic-data-gensql.csv\n" + cmd: "mkdir -p data/discretized/ && python scripts/discretize.py --real data/ignored.csv + --synthetic data/synthetic-data-gensql.csv --schema data/loom-schema.json --real-disc + data/discretized/ignored.csv --synthetic-disc data/discretized/synthetic-data-gensql.csv\n" deps: - path: data/ignored.csv hash: md5 - md5: 5ffeb41f4182b8f7f062b5f13230557d - size: 77540 + md5: 0f31ce8a5706de864b2c503a540b4ac1 + size: 17911 - path: data/loom-schema.json hash: md5 - md5: 68f027a12982e5d4f01f40fe0450b281 - size: 127 + md5: c7f8e3dcb424006ec9aaf56ddf23de3b + size: 416 - path: data/synthetic-data-gensql.csv hash: md5 - md5: a46403f9fa02b0dc465eac9015bd45d0 - size: 84636 + md5: c0b97869f77a0366fa8c94af8b9e54ed + size: 18352 - path: scripts/discretize.py hash: md5 md5: 413bc37503837d33c7f2ae72359e994a @@ -822,33 +834,33 @@ stages: outs: - path: data/discretized/synthetic-data-gensql.csv hash: md5 - md5: a46403f9fa02b0dc465eac9015bd45d0 - size: 84636 + md5: c0b97869f77a0366fa8c94af8b9e54ed + size: 18352 qc-fidelity-table@synthetic-data-gensql.csv: - cmd: "mkdir -p data/fidelity/ && (LD_DEBUG=libs assess-distance --data-1 data/discretized/ignored.csv\ - \ --data-2 data/discretized/synthetic-data-gensql.csv --bivariate > data/fidelity/synthetic-data-gensql.csv)\n" + cmd: "mkdir -p data/fidelity/ && assess-distance --data-1 data/discretized/ignored.csv + --data-2 data/discretized/synthetic-data-gensql.csv --bivariate > data/fidelity/synthetic-data-gensql.csv\n" deps: - path: data/discretized/ignored.csv hash: md5 - md5: 5ffeb41f4182b8f7f062b5f13230557d - size: 77540 + md5: 0f31ce8a5706de864b2c503a540b4ac1 + size: 17911 - path: data/discretized/synthetic-data-gensql.csv hash: md5 - md5: a46403f9fa02b0dc465eac9015bd45d0 - size: 84636 + md5: c0b97869f77a0366fa8c94af8b9e54ed + size: 18352 outs: - path: data/fidelity/synthetic-data-gensql.csv hash: md5 - md5: 4b0979a29feceb30a26e267de1df19f6 - size: 735 + md5: bb55d35b13ec664ced7000e7dedb5a26 + size: 3473 qc-fidelity-json: cmd: "python scripts/fidelity2json.py --params params.yaml >> data/fidelity.json\n" deps: - path: data/fidelity/ hash: md5 - md5: ee1f5499c3c52f3af72097a8dd5f0051.dir - size: 762 - nfiles: 2 + md5: d751713988987e9331980363e24189ce.dir + size: 0 + nfiles: 0 - path: scripts/fidelity2json.py hash: md5 md5: ebe89a5b3530a60999cc3a7c0cfc1493 @@ -857,19 +869,22 @@ stages: params.yaml: synthetic_data_evaluation.datasets: - synthetic-data-gensql.csv + - synthetic-data-gan.csv + - synthetic-data-copula.csv outs: - path: data/fidelity.json hash: md5 - md5: a53aad464233582b9b8841c3a270df01 - size: 1494 + md5: 5d96ff3aff4224888111305f92adfd9e + size: 15954 qc-fidelity-app: - cmd: "sed \"s|{{DATA}}|$(cat data/fidelity.json)|\" qc/templates/fidelity.html\ - \ > qc/app/fidelity.html\n" + cmd: "awk 'BEGIN {data = ARGV[1]; ARGV[1] = \"\"; while ((getline line < data) + > 0) data_str = data_str line \"\\n\"} {gsub(\"{{DATA}}\", data_str)} 1' data/fidelity.json + qc/templates/fidelity.html > qc/app/fidelity.html\n" deps: - path: data/fidelity.json hash: md5 - md5: a53aad464233582b9b8841c3a270df01 - size: 1494 + md5: 5d96ff3aff4224888111305f92adfd9e + size: 15954 - path: qc/templates/fidelity.html hash: md5 md5: 26cb40e5e0c02c8ca73e3dcdaa21f814 @@ -877,5 +892,91 @@ stages: outs: - path: qc/app/fidelity.html hash: md5 - md5: 5d563d763db76aaf9686fb4a4cd582ca - size: 3522 + md5: 2dd6cb406973271fa9758e3efa3a4d76 + size: 17983 + qc-discretize@synthetic-data-gan.csv: + cmd: "mkdir -p data/discretized/ && python scripts/discretize.py --real data/ignored.csv + --synthetic data/synthetic-data-gan.csv --schema data/loom-schema.json --real-disc + data/discretized/ignored.csv --synthetic-disc data/discretized/synthetic-data-gan.csv\n" + deps: + - path: data/ignored.csv + hash: md5 + md5: 0f31ce8a5706de864b2c503a540b4ac1 + size: 17911 + - path: data/loom-schema.json + hash: md5 + md5: c7f8e3dcb424006ec9aaf56ddf23de3b + size: 416 + - path: data/synthetic-data-gan.csv + hash: md5 + md5: cc67facc6f9864cce1a04263b1bba3dc + size: 17991 + - path: scripts/discretize.py + hash: md5 + md5: 413bc37503837d33c7f2ae72359e994a + size: 2033 + outs: + - path: data/discretized/synthetic-data-gan.csv + hash: md5 + md5: cc67facc6f9864cce1a04263b1bba3dc + size: 17991 + qc-discretize@synthetic-data-copula.csv: + cmd: "mkdir -p data/discretized/ && python scripts/discretize.py --real data/ignored.csv + --synthetic data/synthetic-data-copula.csv --schema data/loom-schema.json --real-disc + data/discretized/ignored.csv --synthetic-disc data/discretized/synthetic-data-copula.csv\n" + deps: + - path: data/ignored.csv + hash: md5 + md5: 0f31ce8a5706de864b2c503a540b4ac1 + size: 17911 + - path: data/loom-schema.json + hash: md5 + md5: c7f8e3dcb424006ec9aaf56ddf23de3b + size: 416 + - path: data/synthetic-data-copula.csv + hash: md5 + md5: cd90392877bb60445bc9d14d0729fb61 + size: 17943 + - path: scripts/discretize.py + hash: md5 + md5: 413bc37503837d33c7f2ae72359e994a + size: 2033 + outs: + - path: data/discretized/synthetic-data-copula.csv + hash: md5 + md5: cd90392877bb60445bc9d14d0729fb61 + size: 17943 + qc-fidelity-table@synthetic-data-gan.csv: + cmd: "mkdir -p data/fidelity/ && assess-distance --data-1 data/discretized/ignored.csv + --data-2 data/discretized/synthetic-data-gan.csv --bivariate > data/fidelity/synthetic-data-gan.csv\n" + deps: + - path: data/discretized/ignored.csv + hash: md5 + md5: 0f31ce8a5706de864b2c503a540b4ac1 + size: 17911 + - path: data/discretized/synthetic-data-gan.csv + hash: md5 + md5: cc67facc6f9864cce1a04263b1bba3dc + size: 17991 + outs: + - path: data/fidelity/synthetic-data-gan.csv + hash: md5 + md5: df717f41312beb4fc878a9e76fa94b7f + size: 3449 + qc-fidelity-table@synthetic-data-copula.csv: + cmd: "mkdir -p data/fidelity/ && assess-distance --data-1 data/discretized/ignored.csv + --data-2 data/discretized/synthetic-data-copula.csv --bivariate > data/fidelity/synthetic-data-copula.csv\n" + deps: + - path: data/discretized/ignored.csv + hash: md5 + md5: 0f31ce8a5706de864b2c503a540b4ac1 + size: 17911 + - path: data/discretized/synthetic-data-copula.csv + hash: md5 + md5: cd90392877bb60445bc9d14d0729fb61 + size: 17943 + outs: + - path: data/fidelity/synthetic-data-copula.csv + hash: md5 + md5: 798c73d18e886867ae8a06021cca7c77 + size: 3492 diff --git a/params.yaml b/params.yaml index 23631a0..94076fd 100644 --- a/params.yaml +++ b/params.yaml @@ -1,5 +1,5 @@ seed: 0 -sample_count: 3 +sample_count: 1 sub_sample: #according_to_columns: # Class_of_Orbit: @@ -15,21 +15,40 @@ parallel: nullify: # Entries added here will be treated as null. For example, to treat "NaN" and # "missing" as null uncomment the following two lines: - # - NaN - # - missing + - NaN + - missing schema: # Key/value pairs can be added here to explicitly override the inferred schema # for each column. The available types are as follows: numerical, nominal, and ignore. # For example, to force the "age" column to be treated as a # numerical uncomment the following line: - # age: numerical - # Apogee_km: numerical - # Perigee_km: numerical + + # For nominal variables: + # (a) less than or equal to 10% + # (b) (10%, 20%] + # (c) (20%,30%] + # (d) (30%, 40%] + # (e) (40%, 50%] + # (f) (50%, 60%] + # (g) (60%, 70%] + # (h) (70%, 80%] + # (i) (80%, 90%] + # (j) (90%, 100%] + Municipality: ignore + Total_population__Both_sexes_: nominal + Taxable_income__million_yen_: nominal + Ordinal_Number_of_employed_persons: nominal + Number_of_households__Total_: nominal + Ordinal_Number_of_physicians: nominal + Number_of_dwellings_with_occupying_households: nominal + Ordinal_Total_population__65_and_over_: nominal + Number_of_elderly_nursing_facilities__Questionnaire_about_basic_matters_: nominal + Number_of_lower_secondary_school_students: nominal loom: - extra_passes: 1 + extra_passes: 100 cgpm: iterations: 1 - minutes: 1 + minutes: 450 #dependence: # # While the CrossCat implementation in CGPM takes dependence constraints, doing so # # throws a not-implemented-error. Hence, we apply a workaround: @@ -71,8 +90,8 @@ synthetic_data_evaluation: #N: 10000 # Subsample held-out dataframe with 1000 samples datasets: # Add a list of csv synthetic data files - synthetic-data-gensql.csv # this one is generated by default. - #- synthetic-data-gan.csv - #- synthetic-data-copula.csv + - synthetic-data-gan.csv + - synthetic-data-copula.csv database: table_name: data table_path: data/nullified.csv diff --git a/sdv_generate.py b/sdv_generate.py new file mode 100644 index 0000000..395269c --- /dev/null +++ b/sdv_generate.py @@ -0,0 +1,20 @@ +import pandas as pd + +from sdv.metadata import SingleTableMetadata +from sdv.single_table import GaussianCopulaSynthesizer +from sdv.single_table import CTGANSynthesizer + +df = pd.read_csv("data/ignored.csv") + +metadata = SingleTableMetadata() +metadata.detect_from_dataframe(data=df) + +gan = CTGANSynthesizer(metadata) +gan.fit(df) +gan_df = gan.sample(len(df)) +gan_df.to_csv("data/synthetic-data-gan.csv", index=False) + +copula = GaussianCopulaSynthesizer(metadata) +copula.fit(df) +cop_df = copula.sample(len(df)) +cop_df.to_csv("data/synthetic-data-copula.csv", index=False)