diff --git a/dataeng/Level1/demo-data/1000.csv b/dataeng/Level1/demo-data/1000.csv new file mode 100644 index 00000000..afd57739 --- /dev/null +++ b/dataeng/Level1/demo-data/1000.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Susan, Lee, 612302400000 diff --git a/dataeng/Level1/demo-data/1000.png b/dataeng/Level1/demo-data/1000.png new file mode 100644 index 00000000..b319625e Binary files /dev/null and b/dataeng/Level1/demo-data/1000.png differ diff --git a/dataeng/Level1/demo-data/1001.csv b/dataeng/Level1/demo-data/1001.csv new file mode 100644 index 00000000..d6c5c10c --- /dev/null +++ b/dataeng/Level1/demo-data/1001.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Rosa, Garcia, 670626000000 diff --git a/dataeng/Level1/demo-data/1001.png b/dataeng/Level1/demo-data/1001.png new file mode 100644 index 00000000..0a6f7bf0 Binary files /dev/null and b/dataeng/Level1/demo-data/1001.png differ diff --git a/dataeng/Level1/demo-data/1002.csv b/dataeng/Level1/demo-data/1002.csv new file mode 100644 index 00000000..f74740fe --- /dev/null +++ b/dataeng/Level1/demo-data/1002.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Nathan, Emery, 697237200000 diff --git a/dataeng/Level1/demo-data/1002.png b/dataeng/Level1/demo-data/1002.png new file mode 100644 index 00000000..6db6a785 Binary files /dev/null and b/dataeng/Level1/demo-data/1002.png differ diff --git a/dataeng/Level1/demo-data/1003.csv b/dataeng/Level1/demo-data/1003.csv new file mode 100644 index 00000000..da0e51ea --- /dev/null +++ b/dataeng/Level1/demo-data/1003.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Vernon, Evans, 633733200000 diff --git a/dataeng/Level1/demo-data/1003.png b/dataeng/Level1/demo-data/1003.png new file mode 100644 index 00000000..9a2eae48 Binary files /dev/null and b/dataeng/Level1/demo-data/1003.png differ diff --git a/dataeng/Level1/demo-data/1004.csv b/dataeng/Level1/demo-data/1004.csv new file mode 100644 index 00000000..f32edb85 --- /dev/null +++ b/dataeng/Level1/demo-data/1004.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Maria, Crooks, 1051214400000 diff --git a/dataeng/Level1/demo-data/1004.png b/dataeng/Level1/demo-data/1004.png new file mode 100644 index 00000000..52951dfa Binary files /dev/null and b/dataeng/Level1/demo-data/1004.png differ diff --git a/dataeng/Level1/demo-data/1005.csv b/dataeng/Level1/demo-data/1005.csv new file mode 100644 index 00000000..38a14b90 --- /dev/null +++ b/dataeng/Level1/demo-data/1005.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Sharon, Lang, 700606800000 diff --git a/dataeng/Level1/demo-data/1005.png b/dataeng/Level1/demo-data/1005.png new file mode 100644 index 00000000..0ad02751 Binary files /dev/null and b/dataeng/Level1/demo-data/1005.png differ diff --git a/dataeng/Level1/demo-data/1006.csv b/dataeng/Level1/demo-data/1006.csv new file mode 100644 index 00000000..9ead2fd8 --- /dev/null +++ b/dataeng/Level1/demo-data/1006.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Rick, Mayfield, 412549200000 diff --git a/dataeng/Level1/demo-data/1006.png b/dataeng/Level1/demo-data/1006.png new file mode 100644 index 00000000..b75b18fd Binary files /dev/null and b/dataeng/Level1/demo-data/1006.png differ diff --git a/dataeng/Level1/demo-output/output.csv b/dataeng/Level1/demo-output/output.csv new file mode 100644 index 00000000..a5755ad5 --- /dev/null +++ b/dataeng/Level1/demo-output/output.csv @@ -0,0 +1,8 @@ +user_id, first_name, last_name, birthts, img_path +1006, Rick, Mayfield, 412549200000, /home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/demo-data/1006.png +1003, Vernon, Evans, 633733200000, /home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/demo-data/1003.png +1002, Nathan, Emery, 697237200000, /home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/demo-data/1002.png +1004, Maria, Crooks, 1051214400000, /home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/demo-data/1004.png +1001, Rosa, Garcia, 670626000000, /home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/demo-data/1001.png +1005, Sharon, Lang, 700606800000, /home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/demo-data/1005.png +1000, Susan, Lee, 612302400000, /home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/demo-data/1000.png diff --git a/dataeng/Level1/image_path_finder.py b/dataeng/Level1/image_path_finder.py new file mode 100644 index 00000000..9f01928d --- /dev/null +++ b/dataeng/Level1/image_path_finder.py @@ -0,0 +1,88 @@ +import pathlib +from util import * + + +def append_to_csv(output_file, user_id: int, path_to_csv: str, user_png: bool) -> bool: + """ + Given the path to the .csv file and the user id, create the data needed and then + append it to output_file/output.csv. + :param output_file: path to where the output file should be. + :param user_id: user id to be added to the output file. + :param path_to_csv: path to the .csv file. + :param user_png: is there a photo for this .csv file. + :return: true if the output.csv was changed, false if not. + """ + lines = get_csv(output_file + "/output.csv") + if not len(lines): + lines.append(["user_id", "first_name", "last_name", "birthts", "img_path"]) + if user_png: + jpg_path = path_to_csv[:-4] + jpg_path += ".png" + else: + jpg_path = "None" + temp = get_csv(path_to_csv) + line_to_write = [str(user_id), *temp[1], jpg_path] + for i in range(len(lines)): + if lines[i][0] == line_to_write[0]: + if lines[i] == line_to_write: + return False + else: + lines[i] = line_to_write + break + else: + lines.append(line_to_write) + w = csv.writer(open(output_file + "/output_temp.csv", "w+")) + w.writerows(lines) + os.remove(output_file + "/output.csv") + os.rename(output_file + "/output_temp.csv", output_file + "/output.csv") + return True + + +def find_png_and_id(input_path: str) -> (bool, int): + """ + Takes a path to a .csv file and checks if there's a .png file with the same name in the same directory. + if the file exists then returns the name of the file (since this is also the user id), if it doesn't exist + return -1. + :param input_path: path to the .csv file. + :return: user id if the file exists or -1 otherwise. + """ + try: + user_id = int(pathlib.Path(input_path).stem) + except ValueError: + return False, -1 + input_path = input_path[:-4] + input_path += ".png" + if os.path.isfile(input_path): + return True, user_id + return False, user_id + + +def process(input_path: str, output_path: str) -> (int, list): + """ + Reads all files in input_path (absolute path). looks for a .png and a .csv files that have the same name and + combines them. Stores output in output_path/output.csv. + Returns the number of files found and their names. + :param input_path: path used for finding the input. + :param output_path: path used for finding the output file. + :return: number of files found and their names. + """ + if not os.path.exists(os.path.dirname(input_path)): + raise ValueError("This directory doesn't exist") + count = 0 + filenames = [] + for filename in sorted(os.listdir(input_path)): + if filename.endswith('.csv'): + user_id = find_png_and_id(input_path + "/" + filename) + if user_id[1] == -1: + continue + if append_to_csv(output_path, user_id[1], input_path + "/" + filename, user_id[0]): + count += 1 + filenames.append(pathlib.Path(filename).stem) + filenames.sort() + return count, filenames + + +if __name__ == "__main__": + istr = input("Enter the input absolute path: ") + ostr = input("Enter the output absolute path: ") + print(process(istr, ostr)) diff --git a/dataeng/Level1/processed_data/output.csv b/dataeng/Level1/processed_data/output.csv new file mode 100644 index 00000000..3dc79e76 --- /dev/null +++ b/dataeng/Level1/processed_data/output.csv @@ -0,0 +1,101 @@ +user_id,first_name,last_name,birthts,img_path +1000,Susan,Lee,612302400000,None +1001,Rosa,Garcia,670626000000,None +1002,Nathan,Emery,697237200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1002.png +1003,Vernon,Evans,633733200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1003.png +1004,Maria,Crooks,1051214400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1004.png +1005,Sharon,Lang,700606800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1005.png +1006,Rick,Mayfield,412549200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1006.png +1007,Eddie,Baumbach,874526400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1007.png +1008,Coletta,Haring,768081600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1008.png +1009,Margaret,Smith,388612800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1009.png +1010,James,Martin,879627600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1010.png +1011,Glen,Martin,636066000000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1011.png +1012,Debbie,Mooney,423086400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1012.png +1013,Shaneka,Cook,752533200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1013.png +1014,Kim,Moore,403218000000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1014.png +1015,Lorraine,Bingham,891288000000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1015.png +1016,Betty,Partin,467240400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1016.png +1017,Lewis,Erickson,642283200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1017.png +1018,Jenee,Crouse,429566400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1018.png +1019,Wendy,Jarrette,650145600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1019.png +1020,Becky,Hanner,883429200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1020.png +1021,Ray,Nunes,352501200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1021.png +1022,John,Rose,919630800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1022.png +1023,Michael,Diaz,630277200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1023.png +1024,William,Quick,340059600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1024.png +1025,Dino,Miller,1074805200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1025.png +1026,James,Henry,954705600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1026.png +1027,Gregory,Pittman,398894400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1027.png +1028,Martha,Burwell,503269200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1028.png +1029,Jimmy,Deane,947365200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1029.png +1030,Kathleen,Gittens,921272400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1030.png +1031,Sidney,Woods,865972800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1031.png +1032,Robert,David,666997200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1032.png +1033,Jeanne,Shoup,1018468800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1033.png +1034,Brenda,Croyle,788302800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1034.png +1035,Adolph,Raymond,955915200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1035.png +1036,Ellen,Folkers,821394000000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1036.png +1037,Patrick,Newell,853794000000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1037.png +1038,Mary,Roman,750891600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1038.png +1039,Danielle,Robinson,887230800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1039.png +1040,Clara,Kimble,765403200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1040.png +1041,David,Bailey,1062446400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1041.png +1042,James,Nance,1063051200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1042.png +1043,John,Ramage,378507600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1043.png +1044,Christopher,Hardin,785883600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1044.png +1045,Suzanne,Ferguson,812581200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1045.png +1046,Margaret,Diaz,942872400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1046.png +1047,Kenneth,Osborn,793918800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1047.png +1048,Larry,Sanchez,1068930000000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1048.png +1049,Brian,Brown,814050000000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1049.png +1050,Warren,Green,443048400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1050.png +1051,Janet,Chavarria,1087588800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1051.png +1052,Carolyn,Seymore,385851600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1052.png +1053,Dorothy,Massey,356990400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1053.png +1054,Melissa,Denny,482011200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1054.png +1055,Jaclyn,Adams,614635200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1055.png +1056,Frances,Visher,906580800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1056.png +1057,Chad,Miller,971208000000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1057.png +1058,Olivia,Stout,690674400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1058.png +1059,Wallace,Miller,821134800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1059.png +1060,Judith,Hatchett,925070400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1060.png +1061,Madelyn,Oliver,936907200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1061.png +1062,Robert,Kirk,506552400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1062.png +1063,Stephanie,Nelson,476571600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1063.png +1064,Barbara,Blankenship,655506000000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1064.png +1065,Matthew,Bloom,531176400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1065.png +1066,Jeffrey,Mcdonald,686091600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1066.png +1067,Joseph,Carter,1081713600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1067.png +1068,Eloy,Florentino,848005200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1068.png +1069,David,Myles,432676800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1069.png +1070,Elizabeth,Hoover,334702800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1070.png +1071,Jerome,Weaver,1008795600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1071.png +1072,Morris,Hall,407019600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1072.png +1073,Myrtle,Ferro,840830400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1073.png +1074,Gisela,Patterson,464990400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1074.png +1075,Stacey,Bishop,493848000000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1075.png +1076,Ray,Goins,933451200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1076.png +1077,Blake,Dickerson,508107600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1077.png +1078,David,May,601074000000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1078.png +1079,Cindy,Wilcox,1021060800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1079.png +1080,Mary,Nelson,518644800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1080.png +1081,Amy,Smith,785710800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1081.png +1082,Joseph,Denton,463089600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1082.png +1083,Keith,Wiggin,461707200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1083.png +1084,Jody,Hochmuth,752446800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1084.png +1085,Beverly,Livesey,739224000000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1085.png +1086,Nicole,Pharr,884552400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1086.png +1087,Aaron,Fox,1090094400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1087.png +1088,Rachel,Robinson,528148800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1088.png +1089,Leah,Costales,1072472400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1089.png +1090,Thomas,Williams,946501200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1090.png +1091,Opal,Morse,463780800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1091.png +1092,Alan,Mcgraw,469054800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1092.png +1093,Aaron,Towe,877809600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1093.png +1094,Cecil,Struck,416437200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1094.png +1095,Corey,Davis,1054411200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1095.png +1096,James,Peacock,503442000000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1096.png +1097,John,Bausch,1013029200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1097.png +1098,Maria,Horton,702504000000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1098.png +1099,Samuel,Ladner,815259600000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/src-data/1099.png diff --git a/dataeng/Level1/src-data/1000.csv b/dataeng/Level1/src-data/1000.csv new file mode 100644 index 00000000..afd57739 --- /dev/null +++ b/dataeng/Level1/src-data/1000.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Susan, Lee, 612302400000 diff --git a/dataeng/Level1/src-data/1000.png b/dataeng/Level1/src-data/1000.png new file mode 100644 index 00000000..b319625e Binary files /dev/null and b/dataeng/Level1/src-data/1000.png differ diff --git a/dataeng/Level1/src-data/1001.csv b/dataeng/Level1/src-data/1001.csv new file mode 100644 index 00000000..d6c5c10c --- /dev/null +++ b/dataeng/Level1/src-data/1001.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Rosa, Garcia, 670626000000 diff --git a/dataeng/Level1/src-data/1001.png b/dataeng/Level1/src-data/1001.png new file mode 100644 index 00000000..0a6f7bf0 Binary files /dev/null and b/dataeng/Level1/src-data/1001.png differ diff --git a/dataeng/Level1/src-data/1002.csv b/dataeng/Level1/src-data/1002.csv new file mode 100644 index 00000000..f74740fe --- /dev/null +++ b/dataeng/Level1/src-data/1002.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Nathan, Emery, 697237200000 diff --git a/dataeng/Level1/src-data/1002.png b/dataeng/Level1/src-data/1002.png new file mode 100644 index 00000000..6db6a785 Binary files /dev/null and b/dataeng/Level1/src-data/1002.png differ diff --git a/dataeng/Level1/src-data/1003.csv b/dataeng/Level1/src-data/1003.csv new file mode 100644 index 00000000..da0e51ea --- /dev/null +++ b/dataeng/Level1/src-data/1003.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Vernon, Evans, 633733200000 diff --git a/dataeng/Level1/src-data/1003.png b/dataeng/Level1/src-data/1003.png new file mode 100644 index 00000000..9a2eae48 Binary files /dev/null and b/dataeng/Level1/src-data/1003.png differ diff --git a/dataeng/Level1/src-data/1004.csv b/dataeng/Level1/src-data/1004.csv new file mode 100644 index 00000000..f32edb85 --- /dev/null +++ b/dataeng/Level1/src-data/1004.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Maria, Crooks, 1051214400000 diff --git a/dataeng/Level1/src-data/1004.png b/dataeng/Level1/src-data/1004.png new file mode 100644 index 00000000..52951dfa Binary files /dev/null and b/dataeng/Level1/src-data/1004.png differ diff --git a/dataeng/Level1/src-data/1005.csv b/dataeng/Level1/src-data/1005.csv new file mode 100644 index 00000000..38a14b90 --- /dev/null +++ b/dataeng/Level1/src-data/1005.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Sharon, Lang, 700606800000 diff --git a/dataeng/Level1/src-data/1005.png b/dataeng/Level1/src-data/1005.png new file mode 100644 index 00000000..0ad02751 Binary files /dev/null and b/dataeng/Level1/src-data/1005.png differ diff --git a/dataeng/Level1/src-data/1006.csv b/dataeng/Level1/src-data/1006.csv new file mode 100644 index 00000000..9ead2fd8 --- /dev/null +++ b/dataeng/Level1/src-data/1006.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Rick, Mayfield, 412549200000 diff --git a/dataeng/Level1/src-data/1006.png b/dataeng/Level1/src-data/1006.png new file mode 100644 index 00000000..b75b18fd Binary files /dev/null and b/dataeng/Level1/src-data/1006.png differ diff --git a/dataeng/Level1/src-data/1007.csv b/dataeng/Level1/src-data/1007.csv new file mode 100644 index 00000000..07ef290c --- /dev/null +++ b/dataeng/Level1/src-data/1007.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Eddie, Baumbach, 874526400000 diff --git a/dataeng/Level1/src-data/1007.png b/dataeng/Level1/src-data/1007.png new file mode 100644 index 00000000..6f6f64f3 Binary files /dev/null and b/dataeng/Level1/src-data/1007.png differ diff --git a/dataeng/Level1/src-data/1008.csv b/dataeng/Level1/src-data/1008.csv new file mode 100644 index 00000000..6dea1c5e --- /dev/null +++ b/dataeng/Level1/src-data/1008.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Coletta, Haring, 768081600000 diff --git a/dataeng/Level1/src-data/1008.png b/dataeng/Level1/src-data/1008.png new file mode 100644 index 00000000..39f19094 Binary files /dev/null and b/dataeng/Level1/src-data/1008.png differ diff --git a/dataeng/Level1/src-data/1009.csv b/dataeng/Level1/src-data/1009.csv new file mode 100644 index 00000000..3bc3e597 --- /dev/null +++ b/dataeng/Level1/src-data/1009.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Margaret, Smith, 388612800000 diff --git a/dataeng/Level1/src-data/1009.png b/dataeng/Level1/src-data/1009.png new file mode 100644 index 00000000..d443452b Binary files /dev/null and b/dataeng/Level1/src-data/1009.png differ diff --git a/dataeng/Level1/src-data/1010.csv b/dataeng/Level1/src-data/1010.csv new file mode 100644 index 00000000..6be637ad --- /dev/null +++ b/dataeng/Level1/src-data/1010.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +James, Martin, 879627600000 diff --git a/dataeng/Level1/src-data/1010.png b/dataeng/Level1/src-data/1010.png new file mode 100644 index 00000000..0df6619d Binary files /dev/null and b/dataeng/Level1/src-data/1010.png differ diff --git a/dataeng/Level1/src-data/1011.csv b/dataeng/Level1/src-data/1011.csv new file mode 100644 index 00000000..380495e7 --- /dev/null +++ b/dataeng/Level1/src-data/1011.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Glen, Martin, 636066000000 diff --git a/dataeng/Level1/src-data/1011.png b/dataeng/Level1/src-data/1011.png new file mode 100644 index 00000000..c9970adb Binary files /dev/null and b/dataeng/Level1/src-data/1011.png differ diff --git a/dataeng/Level1/src-data/1012.csv b/dataeng/Level1/src-data/1012.csv new file mode 100644 index 00000000..b3bbe8da --- /dev/null +++ b/dataeng/Level1/src-data/1012.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Debbie, Mooney, 423086400000 diff --git a/dataeng/Level1/src-data/1012.png b/dataeng/Level1/src-data/1012.png new file mode 100644 index 00000000..d45bf51f Binary files /dev/null and b/dataeng/Level1/src-data/1012.png differ diff --git a/dataeng/Level1/src-data/1013.csv b/dataeng/Level1/src-data/1013.csv new file mode 100644 index 00000000..a1a422ae --- /dev/null +++ b/dataeng/Level1/src-data/1013.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Shaneka, Cook, 752533200000 diff --git a/dataeng/Level1/src-data/1013.png b/dataeng/Level1/src-data/1013.png new file mode 100644 index 00000000..9cbe8e12 Binary files /dev/null and b/dataeng/Level1/src-data/1013.png differ diff --git a/dataeng/Level1/src-data/1014.csv b/dataeng/Level1/src-data/1014.csv new file mode 100644 index 00000000..ed86a7d1 --- /dev/null +++ b/dataeng/Level1/src-data/1014.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Kim, Moore, 403218000000 diff --git a/dataeng/Level1/src-data/1014.png b/dataeng/Level1/src-data/1014.png new file mode 100644 index 00000000..c34b9954 Binary files /dev/null and b/dataeng/Level1/src-data/1014.png differ diff --git a/dataeng/Level1/src-data/1015.csv b/dataeng/Level1/src-data/1015.csv new file mode 100644 index 00000000..e7f4f92d --- /dev/null +++ b/dataeng/Level1/src-data/1015.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Lorraine, Bingham, 891288000000 diff --git a/dataeng/Level1/src-data/1015.png b/dataeng/Level1/src-data/1015.png new file mode 100644 index 00000000..79e4cb66 Binary files /dev/null and b/dataeng/Level1/src-data/1015.png differ diff --git a/dataeng/Level1/src-data/1016.csv b/dataeng/Level1/src-data/1016.csv new file mode 100644 index 00000000..c08470fd --- /dev/null +++ b/dataeng/Level1/src-data/1016.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Betty, Partin, 467240400000 diff --git a/dataeng/Level1/src-data/1016.png b/dataeng/Level1/src-data/1016.png new file mode 100644 index 00000000..e929ea78 Binary files /dev/null and b/dataeng/Level1/src-data/1016.png differ diff --git a/dataeng/Level1/src-data/1017.csv b/dataeng/Level1/src-data/1017.csv new file mode 100644 index 00000000..8ae81034 --- /dev/null +++ b/dataeng/Level1/src-data/1017.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Lewis, Erickson, 642283200000 diff --git a/dataeng/Level1/src-data/1017.png b/dataeng/Level1/src-data/1017.png new file mode 100644 index 00000000..f20ea9eb Binary files /dev/null and b/dataeng/Level1/src-data/1017.png differ diff --git a/dataeng/Level1/src-data/1018.csv b/dataeng/Level1/src-data/1018.csv new file mode 100644 index 00000000..a8f8b2fd --- /dev/null +++ b/dataeng/Level1/src-data/1018.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Jenee, Crouse, 429566400000 diff --git a/dataeng/Level1/src-data/1018.png b/dataeng/Level1/src-data/1018.png new file mode 100644 index 00000000..59fd4b1b Binary files /dev/null and b/dataeng/Level1/src-data/1018.png differ diff --git a/dataeng/Level1/src-data/1019.csv b/dataeng/Level1/src-data/1019.csv new file mode 100644 index 00000000..8fd5eefd --- /dev/null +++ b/dataeng/Level1/src-data/1019.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Wendy, Jarrette, 650145600000 diff --git a/dataeng/Level1/src-data/1019.png b/dataeng/Level1/src-data/1019.png new file mode 100644 index 00000000..7f45462b Binary files /dev/null and b/dataeng/Level1/src-data/1019.png differ diff --git a/dataeng/Level1/src-data/1020.csv b/dataeng/Level1/src-data/1020.csv new file mode 100644 index 00000000..bb6ef9cc --- /dev/null +++ b/dataeng/Level1/src-data/1020.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Becky, Hanner, 883429200000 diff --git a/dataeng/Level1/src-data/1020.png b/dataeng/Level1/src-data/1020.png new file mode 100644 index 00000000..f9f41080 Binary files /dev/null and b/dataeng/Level1/src-data/1020.png differ diff --git a/dataeng/Level1/src-data/1021.csv b/dataeng/Level1/src-data/1021.csv new file mode 100644 index 00000000..d995050c --- /dev/null +++ b/dataeng/Level1/src-data/1021.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Ray, Nunes, 352501200000 diff --git a/dataeng/Level1/src-data/1021.png b/dataeng/Level1/src-data/1021.png new file mode 100644 index 00000000..a89a33b1 Binary files /dev/null and b/dataeng/Level1/src-data/1021.png differ diff --git a/dataeng/Level1/src-data/1022.csv b/dataeng/Level1/src-data/1022.csv new file mode 100644 index 00000000..e7b26375 --- /dev/null +++ b/dataeng/Level1/src-data/1022.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +John, Rose, 919630800000 diff --git a/dataeng/Level1/src-data/1022.png b/dataeng/Level1/src-data/1022.png new file mode 100644 index 00000000..3d39d092 Binary files /dev/null and b/dataeng/Level1/src-data/1022.png differ diff --git a/dataeng/Level1/src-data/1023.csv b/dataeng/Level1/src-data/1023.csv new file mode 100644 index 00000000..cb7037f7 --- /dev/null +++ b/dataeng/Level1/src-data/1023.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Michael, Diaz, 630277200000 diff --git a/dataeng/Level1/src-data/1023.png b/dataeng/Level1/src-data/1023.png new file mode 100644 index 00000000..c965dbcc Binary files /dev/null and b/dataeng/Level1/src-data/1023.png differ diff --git a/dataeng/Level1/src-data/1024.csv b/dataeng/Level1/src-data/1024.csv new file mode 100644 index 00000000..b648b695 --- /dev/null +++ b/dataeng/Level1/src-data/1024.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +William, Quick, 340059600000 diff --git a/dataeng/Level1/src-data/1024.png b/dataeng/Level1/src-data/1024.png new file mode 100644 index 00000000..07312235 Binary files /dev/null and b/dataeng/Level1/src-data/1024.png differ diff --git a/dataeng/Level1/src-data/1025.csv b/dataeng/Level1/src-data/1025.csv new file mode 100644 index 00000000..48fcc892 --- /dev/null +++ b/dataeng/Level1/src-data/1025.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Dino, Miller, 1074805200000 diff --git a/dataeng/Level1/src-data/1025.png b/dataeng/Level1/src-data/1025.png new file mode 100644 index 00000000..f0c32450 Binary files /dev/null and b/dataeng/Level1/src-data/1025.png differ diff --git a/dataeng/Level1/src-data/1026.csv b/dataeng/Level1/src-data/1026.csv new file mode 100644 index 00000000..425c86fb --- /dev/null +++ b/dataeng/Level1/src-data/1026.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +James, Henry, 954705600000 diff --git a/dataeng/Level1/src-data/1026.png b/dataeng/Level1/src-data/1026.png new file mode 100644 index 00000000..7754f084 Binary files /dev/null and b/dataeng/Level1/src-data/1026.png differ diff --git a/dataeng/Level1/src-data/1027.csv b/dataeng/Level1/src-data/1027.csv new file mode 100644 index 00000000..66c43641 --- /dev/null +++ b/dataeng/Level1/src-data/1027.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Gregory, Pittman, 398894400000 diff --git a/dataeng/Level1/src-data/1027.png b/dataeng/Level1/src-data/1027.png new file mode 100644 index 00000000..982c7b8e Binary files /dev/null and b/dataeng/Level1/src-data/1027.png differ diff --git a/dataeng/Level1/src-data/1028.csv b/dataeng/Level1/src-data/1028.csv new file mode 100644 index 00000000..f26a0645 --- /dev/null +++ b/dataeng/Level1/src-data/1028.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Martha, Burwell, 503269200000 diff --git a/dataeng/Level1/src-data/1028.png b/dataeng/Level1/src-data/1028.png new file mode 100644 index 00000000..09c1d2c9 Binary files /dev/null and b/dataeng/Level1/src-data/1028.png differ diff --git a/dataeng/Level1/src-data/1029.csv b/dataeng/Level1/src-data/1029.csv new file mode 100644 index 00000000..2fb9c552 --- /dev/null +++ b/dataeng/Level1/src-data/1029.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Jimmy, Deane, 947365200000 diff --git a/dataeng/Level1/src-data/1029.png b/dataeng/Level1/src-data/1029.png new file mode 100644 index 00000000..8db3311d Binary files /dev/null and b/dataeng/Level1/src-data/1029.png differ diff --git a/dataeng/Level1/src-data/1030.csv b/dataeng/Level1/src-data/1030.csv new file mode 100644 index 00000000..2a4bd974 --- /dev/null +++ b/dataeng/Level1/src-data/1030.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Kathleen, Gittens, 921272400000 diff --git a/dataeng/Level1/src-data/1030.png b/dataeng/Level1/src-data/1030.png new file mode 100644 index 00000000..3155d262 Binary files /dev/null and b/dataeng/Level1/src-data/1030.png differ diff --git a/dataeng/Level1/src-data/1031.csv b/dataeng/Level1/src-data/1031.csv new file mode 100644 index 00000000..8c2d2fb6 --- /dev/null +++ b/dataeng/Level1/src-data/1031.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Sidney, Woods, 865972800000 diff --git a/dataeng/Level1/src-data/1031.png b/dataeng/Level1/src-data/1031.png new file mode 100644 index 00000000..19303091 Binary files /dev/null and b/dataeng/Level1/src-data/1031.png differ diff --git a/dataeng/Level1/src-data/1032.csv b/dataeng/Level1/src-data/1032.csv new file mode 100644 index 00000000..2f6cd114 --- /dev/null +++ b/dataeng/Level1/src-data/1032.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Robert, David, 666997200000 diff --git a/dataeng/Level1/src-data/1032.png b/dataeng/Level1/src-data/1032.png new file mode 100644 index 00000000..320c71e2 Binary files /dev/null and b/dataeng/Level1/src-data/1032.png differ diff --git a/dataeng/Level1/src-data/1033.csv b/dataeng/Level1/src-data/1033.csv new file mode 100644 index 00000000..170e4b23 --- /dev/null +++ b/dataeng/Level1/src-data/1033.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Jeanne, Shoup, 1018468800000 diff --git a/dataeng/Level1/src-data/1033.png b/dataeng/Level1/src-data/1033.png new file mode 100644 index 00000000..5b88a67a Binary files /dev/null and b/dataeng/Level1/src-data/1033.png differ diff --git a/dataeng/Level1/src-data/1034.csv b/dataeng/Level1/src-data/1034.csv new file mode 100644 index 00000000..2bd23042 --- /dev/null +++ b/dataeng/Level1/src-data/1034.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Brenda, Croyle, 788302800000 diff --git a/dataeng/Level1/src-data/1034.png b/dataeng/Level1/src-data/1034.png new file mode 100644 index 00000000..7e593293 Binary files /dev/null and b/dataeng/Level1/src-data/1034.png differ diff --git a/dataeng/Level1/src-data/1035.csv b/dataeng/Level1/src-data/1035.csv new file mode 100644 index 00000000..f0ae169f --- /dev/null +++ b/dataeng/Level1/src-data/1035.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Adolph, Raymond, 955915200000 diff --git a/dataeng/Level1/src-data/1035.png b/dataeng/Level1/src-data/1035.png new file mode 100644 index 00000000..d410d044 Binary files /dev/null and b/dataeng/Level1/src-data/1035.png differ diff --git a/dataeng/Level1/src-data/1036.csv b/dataeng/Level1/src-data/1036.csv new file mode 100644 index 00000000..55cb1219 --- /dev/null +++ b/dataeng/Level1/src-data/1036.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Ellen, Folkers, 821394000000 diff --git a/dataeng/Level1/src-data/1036.png b/dataeng/Level1/src-data/1036.png new file mode 100644 index 00000000..41741cc7 Binary files /dev/null and b/dataeng/Level1/src-data/1036.png differ diff --git a/dataeng/Level1/src-data/1037.csv b/dataeng/Level1/src-data/1037.csv new file mode 100644 index 00000000..8008314a --- /dev/null +++ b/dataeng/Level1/src-data/1037.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Patrick, Newell, 853794000000 diff --git a/dataeng/Level1/src-data/1037.png b/dataeng/Level1/src-data/1037.png new file mode 100644 index 00000000..52f6ec41 Binary files /dev/null and b/dataeng/Level1/src-data/1037.png differ diff --git a/dataeng/Level1/src-data/1038.csv b/dataeng/Level1/src-data/1038.csv new file mode 100644 index 00000000..46f3c585 --- /dev/null +++ b/dataeng/Level1/src-data/1038.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Mary, Roman, 750891600000 diff --git a/dataeng/Level1/src-data/1038.png b/dataeng/Level1/src-data/1038.png new file mode 100644 index 00000000..9d5c3142 Binary files /dev/null and b/dataeng/Level1/src-data/1038.png differ diff --git a/dataeng/Level1/src-data/1039.csv b/dataeng/Level1/src-data/1039.csv new file mode 100644 index 00000000..966b62eb --- /dev/null +++ b/dataeng/Level1/src-data/1039.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Danielle, Robinson, 887230800000 diff --git a/dataeng/Level1/src-data/1039.png b/dataeng/Level1/src-data/1039.png new file mode 100644 index 00000000..5c464b81 Binary files /dev/null and b/dataeng/Level1/src-data/1039.png differ diff --git a/dataeng/Level1/src-data/1040.csv b/dataeng/Level1/src-data/1040.csv new file mode 100644 index 00000000..e3bf975a --- /dev/null +++ b/dataeng/Level1/src-data/1040.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Clara, Kimble, 765403200000 diff --git a/dataeng/Level1/src-data/1040.png b/dataeng/Level1/src-data/1040.png new file mode 100644 index 00000000..1c15ec1a Binary files /dev/null and b/dataeng/Level1/src-data/1040.png differ diff --git a/dataeng/Level1/src-data/1041.csv b/dataeng/Level1/src-data/1041.csv new file mode 100644 index 00000000..3c91985c --- /dev/null +++ b/dataeng/Level1/src-data/1041.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +David, Bailey, 1062446400000 diff --git a/dataeng/Level1/src-data/1041.png b/dataeng/Level1/src-data/1041.png new file mode 100644 index 00000000..8f92dc7c Binary files /dev/null and b/dataeng/Level1/src-data/1041.png differ diff --git a/dataeng/Level1/src-data/1042.csv b/dataeng/Level1/src-data/1042.csv new file mode 100644 index 00000000..48dabe3e --- /dev/null +++ b/dataeng/Level1/src-data/1042.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +James, Nance, 1063051200000 diff --git a/dataeng/Level1/src-data/1042.png b/dataeng/Level1/src-data/1042.png new file mode 100644 index 00000000..fdc0818d Binary files /dev/null and b/dataeng/Level1/src-data/1042.png differ diff --git a/dataeng/Level1/src-data/1043.csv b/dataeng/Level1/src-data/1043.csv new file mode 100644 index 00000000..2422d3ea --- /dev/null +++ b/dataeng/Level1/src-data/1043.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +John, Ramage, 378507600000 diff --git a/dataeng/Level1/src-data/1043.png b/dataeng/Level1/src-data/1043.png new file mode 100644 index 00000000..197b47a5 Binary files /dev/null and b/dataeng/Level1/src-data/1043.png differ diff --git a/dataeng/Level1/src-data/1044.csv b/dataeng/Level1/src-data/1044.csv new file mode 100644 index 00000000..a7ce5513 --- /dev/null +++ b/dataeng/Level1/src-data/1044.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Christopher, Hardin, 785883600000 diff --git a/dataeng/Level1/src-data/1044.png b/dataeng/Level1/src-data/1044.png new file mode 100644 index 00000000..3c6e44cf Binary files /dev/null and b/dataeng/Level1/src-data/1044.png differ diff --git a/dataeng/Level1/src-data/1045.csv b/dataeng/Level1/src-data/1045.csv new file mode 100644 index 00000000..0a5901af --- /dev/null +++ b/dataeng/Level1/src-data/1045.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Suzanne, Ferguson, 812581200000 diff --git a/dataeng/Level1/src-data/1045.png b/dataeng/Level1/src-data/1045.png new file mode 100644 index 00000000..f283d4af Binary files /dev/null and b/dataeng/Level1/src-data/1045.png differ diff --git a/dataeng/Level1/src-data/1046.csv b/dataeng/Level1/src-data/1046.csv new file mode 100644 index 00000000..df67fd52 --- /dev/null +++ b/dataeng/Level1/src-data/1046.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Margaret, Diaz, 942872400000 diff --git a/dataeng/Level1/src-data/1046.png b/dataeng/Level1/src-data/1046.png new file mode 100644 index 00000000..c98af3d9 Binary files /dev/null and b/dataeng/Level1/src-data/1046.png differ diff --git a/dataeng/Level1/src-data/1047.csv b/dataeng/Level1/src-data/1047.csv new file mode 100644 index 00000000..14153e11 --- /dev/null +++ b/dataeng/Level1/src-data/1047.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Kenneth, Osborn, 793918800000 diff --git a/dataeng/Level1/src-data/1047.png b/dataeng/Level1/src-data/1047.png new file mode 100644 index 00000000..8d137bee Binary files /dev/null and b/dataeng/Level1/src-data/1047.png differ diff --git a/dataeng/Level1/src-data/1048.csv b/dataeng/Level1/src-data/1048.csv new file mode 100644 index 00000000..ef0c922a --- /dev/null +++ b/dataeng/Level1/src-data/1048.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Larry, Sanchez, 1068930000000 diff --git a/dataeng/Level1/src-data/1048.png b/dataeng/Level1/src-data/1048.png new file mode 100644 index 00000000..c812caae Binary files /dev/null and b/dataeng/Level1/src-data/1048.png differ diff --git a/dataeng/Level1/src-data/1049.csv b/dataeng/Level1/src-data/1049.csv new file mode 100644 index 00000000..b5ddc5e5 --- /dev/null +++ b/dataeng/Level1/src-data/1049.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Brian, Brown, 814050000000 diff --git a/dataeng/Level1/src-data/1049.png b/dataeng/Level1/src-data/1049.png new file mode 100644 index 00000000..d97334c8 Binary files /dev/null and b/dataeng/Level1/src-data/1049.png differ diff --git a/dataeng/Level1/src-data/1050.csv b/dataeng/Level1/src-data/1050.csv new file mode 100644 index 00000000..3cad1e1b --- /dev/null +++ b/dataeng/Level1/src-data/1050.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Warren, Green, 443048400000 diff --git a/dataeng/Level1/src-data/1050.png b/dataeng/Level1/src-data/1050.png new file mode 100644 index 00000000..b3eeb765 Binary files /dev/null and b/dataeng/Level1/src-data/1050.png differ diff --git a/dataeng/Level1/src-data/1051.csv b/dataeng/Level1/src-data/1051.csv new file mode 100644 index 00000000..2a74171e --- /dev/null +++ b/dataeng/Level1/src-data/1051.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Janet, Chavarria, 1087588800000 diff --git a/dataeng/Level1/src-data/1051.png b/dataeng/Level1/src-data/1051.png new file mode 100644 index 00000000..5bc56f98 Binary files /dev/null and b/dataeng/Level1/src-data/1051.png differ diff --git a/dataeng/Level1/src-data/1052.csv b/dataeng/Level1/src-data/1052.csv new file mode 100644 index 00000000..63dc944f --- /dev/null +++ b/dataeng/Level1/src-data/1052.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Carolyn, Seymore, 385851600000 diff --git a/dataeng/Level1/src-data/1052.png b/dataeng/Level1/src-data/1052.png new file mode 100644 index 00000000..788a7f48 Binary files /dev/null and b/dataeng/Level1/src-data/1052.png differ diff --git a/dataeng/Level1/src-data/1053.csv b/dataeng/Level1/src-data/1053.csv new file mode 100644 index 00000000..567623d9 --- /dev/null +++ b/dataeng/Level1/src-data/1053.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Dorothy, Massey, 356990400000 diff --git a/dataeng/Level1/src-data/1053.png b/dataeng/Level1/src-data/1053.png new file mode 100644 index 00000000..b3067c6f Binary files /dev/null and b/dataeng/Level1/src-data/1053.png differ diff --git a/dataeng/Level1/src-data/1054.csv b/dataeng/Level1/src-data/1054.csv new file mode 100644 index 00000000..2560f6c8 --- /dev/null +++ b/dataeng/Level1/src-data/1054.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Melissa, Denny, 482011200000 diff --git a/dataeng/Level1/src-data/1054.png b/dataeng/Level1/src-data/1054.png new file mode 100644 index 00000000..e9dd9846 Binary files /dev/null and b/dataeng/Level1/src-data/1054.png differ diff --git a/dataeng/Level1/src-data/1055.csv b/dataeng/Level1/src-data/1055.csv new file mode 100644 index 00000000..90f06d11 --- /dev/null +++ b/dataeng/Level1/src-data/1055.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Jaclyn, Adams, 614635200000 diff --git a/dataeng/Level1/src-data/1055.png b/dataeng/Level1/src-data/1055.png new file mode 100644 index 00000000..99f75f50 Binary files /dev/null and b/dataeng/Level1/src-data/1055.png differ diff --git a/dataeng/Level1/src-data/1056.csv b/dataeng/Level1/src-data/1056.csv new file mode 100644 index 00000000..6f4b03d8 --- /dev/null +++ b/dataeng/Level1/src-data/1056.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Frances, Visher, 906580800000 diff --git a/dataeng/Level1/src-data/1056.png b/dataeng/Level1/src-data/1056.png new file mode 100644 index 00000000..1459a3f7 Binary files /dev/null and b/dataeng/Level1/src-data/1056.png differ diff --git a/dataeng/Level1/src-data/1057.csv b/dataeng/Level1/src-data/1057.csv new file mode 100644 index 00000000..7a36eace --- /dev/null +++ b/dataeng/Level1/src-data/1057.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Chad, Miller, 971208000000 diff --git a/dataeng/Level1/src-data/1057.png b/dataeng/Level1/src-data/1057.png new file mode 100644 index 00000000..eb378161 Binary files /dev/null and b/dataeng/Level1/src-data/1057.png differ diff --git a/dataeng/Level1/src-data/1058.csv b/dataeng/Level1/src-data/1058.csv new file mode 100644 index 00000000..d8436d70 --- /dev/null +++ b/dataeng/Level1/src-data/1058.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Olivia, Stout, 690674400000 diff --git a/dataeng/Level1/src-data/1058.png b/dataeng/Level1/src-data/1058.png new file mode 100644 index 00000000..de849efc Binary files /dev/null and b/dataeng/Level1/src-data/1058.png differ diff --git a/dataeng/Level1/src-data/1059.csv b/dataeng/Level1/src-data/1059.csv new file mode 100644 index 00000000..2941c61f --- /dev/null +++ b/dataeng/Level1/src-data/1059.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Wallace, Miller, 821134800000 diff --git a/dataeng/Level1/src-data/1059.png b/dataeng/Level1/src-data/1059.png new file mode 100644 index 00000000..3c85f28b Binary files /dev/null and b/dataeng/Level1/src-data/1059.png differ diff --git a/dataeng/Level1/src-data/1060.csv b/dataeng/Level1/src-data/1060.csv new file mode 100644 index 00000000..5daf1476 --- /dev/null +++ b/dataeng/Level1/src-data/1060.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Judith, Hatchett, 925070400000 diff --git a/dataeng/Level1/src-data/1060.png b/dataeng/Level1/src-data/1060.png new file mode 100644 index 00000000..ea3d5ab5 Binary files /dev/null and b/dataeng/Level1/src-data/1060.png differ diff --git a/dataeng/Level1/src-data/1061.csv b/dataeng/Level1/src-data/1061.csv new file mode 100644 index 00000000..6b0a9fbf --- /dev/null +++ b/dataeng/Level1/src-data/1061.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Madelyn, Oliver, 936907200000 diff --git a/dataeng/Level1/src-data/1061.png b/dataeng/Level1/src-data/1061.png new file mode 100644 index 00000000..ed0c3e1d Binary files /dev/null and b/dataeng/Level1/src-data/1061.png differ diff --git a/dataeng/Level1/src-data/1062.csv b/dataeng/Level1/src-data/1062.csv new file mode 100644 index 00000000..09098922 --- /dev/null +++ b/dataeng/Level1/src-data/1062.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Robert, Kirk, 506552400000 diff --git a/dataeng/Level1/src-data/1062.png b/dataeng/Level1/src-data/1062.png new file mode 100644 index 00000000..b98e4fe4 Binary files /dev/null and b/dataeng/Level1/src-data/1062.png differ diff --git a/dataeng/Level1/src-data/1063.csv b/dataeng/Level1/src-data/1063.csv new file mode 100644 index 00000000..2c9a185d --- /dev/null +++ b/dataeng/Level1/src-data/1063.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Stephanie, Nelson, 476571600000 diff --git a/dataeng/Level1/src-data/1063.png b/dataeng/Level1/src-data/1063.png new file mode 100644 index 00000000..f4496298 Binary files /dev/null and b/dataeng/Level1/src-data/1063.png differ diff --git a/dataeng/Level1/src-data/1064.csv b/dataeng/Level1/src-data/1064.csv new file mode 100644 index 00000000..cd331f02 --- /dev/null +++ b/dataeng/Level1/src-data/1064.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Barbara, Blankenship, 655506000000 diff --git a/dataeng/Level1/src-data/1064.png b/dataeng/Level1/src-data/1064.png new file mode 100644 index 00000000..450b1d81 Binary files /dev/null and b/dataeng/Level1/src-data/1064.png differ diff --git a/dataeng/Level1/src-data/1065.csv b/dataeng/Level1/src-data/1065.csv new file mode 100644 index 00000000..7856b1c0 --- /dev/null +++ b/dataeng/Level1/src-data/1065.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Matthew, Bloom, 531176400000 diff --git a/dataeng/Level1/src-data/1065.png b/dataeng/Level1/src-data/1065.png new file mode 100644 index 00000000..d1b097bd Binary files /dev/null and b/dataeng/Level1/src-data/1065.png differ diff --git a/dataeng/Level1/src-data/1066.csv b/dataeng/Level1/src-data/1066.csv new file mode 100644 index 00000000..32b0aca5 --- /dev/null +++ b/dataeng/Level1/src-data/1066.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Jeffrey, Mcdonald, 686091600000 diff --git a/dataeng/Level1/src-data/1066.png b/dataeng/Level1/src-data/1066.png new file mode 100644 index 00000000..87682674 Binary files /dev/null and b/dataeng/Level1/src-data/1066.png differ diff --git a/dataeng/Level1/src-data/1067.csv b/dataeng/Level1/src-data/1067.csv new file mode 100644 index 00000000..4ac985cf --- /dev/null +++ b/dataeng/Level1/src-data/1067.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Joseph, Carter, 1081713600000 diff --git a/dataeng/Level1/src-data/1067.png b/dataeng/Level1/src-data/1067.png new file mode 100644 index 00000000..56b70bb0 Binary files /dev/null and b/dataeng/Level1/src-data/1067.png differ diff --git a/dataeng/Level1/src-data/1068.csv b/dataeng/Level1/src-data/1068.csv new file mode 100644 index 00000000..0340970f --- /dev/null +++ b/dataeng/Level1/src-data/1068.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Eloy, Florentino, 848005200000 diff --git a/dataeng/Level1/src-data/1068.png b/dataeng/Level1/src-data/1068.png new file mode 100644 index 00000000..e3f56067 Binary files /dev/null and b/dataeng/Level1/src-data/1068.png differ diff --git a/dataeng/Level1/src-data/1069.csv b/dataeng/Level1/src-data/1069.csv new file mode 100644 index 00000000..9514cb30 --- /dev/null +++ b/dataeng/Level1/src-data/1069.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +David, Myles, 432676800000 diff --git a/dataeng/Level1/src-data/1069.png b/dataeng/Level1/src-data/1069.png new file mode 100644 index 00000000..1a411428 Binary files /dev/null and b/dataeng/Level1/src-data/1069.png differ diff --git a/dataeng/Level1/src-data/1070.csv b/dataeng/Level1/src-data/1070.csv new file mode 100644 index 00000000..d8319b73 --- /dev/null +++ b/dataeng/Level1/src-data/1070.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Elizabeth, Hoover, 334702800000 diff --git a/dataeng/Level1/src-data/1070.png b/dataeng/Level1/src-data/1070.png new file mode 100644 index 00000000..cc7bcdad Binary files /dev/null and b/dataeng/Level1/src-data/1070.png differ diff --git a/dataeng/Level1/src-data/1071.csv b/dataeng/Level1/src-data/1071.csv new file mode 100644 index 00000000..c9045b2f --- /dev/null +++ b/dataeng/Level1/src-data/1071.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Jerome, Weaver, 1008795600000 diff --git a/dataeng/Level1/src-data/1071.png b/dataeng/Level1/src-data/1071.png new file mode 100644 index 00000000..f004184f Binary files /dev/null and b/dataeng/Level1/src-data/1071.png differ diff --git a/dataeng/Level1/src-data/1072.csv b/dataeng/Level1/src-data/1072.csv new file mode 100644 index 00000000..ef856c34 --- /dev/null +++ b/dataeng/Level1/src-data/1072.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Morris, Hall, 407019600000 diff --git a/dataeng/Level1/src-data/1072.png b/dataeng/Level1/src-data/1072.png new file mode 100644 index 00000000..fe3a33f6 Binary files /dev/null and b/dataeng/Level1/src-data/1072.png differ diff --git a/dataeng/Level1/src-data/1073.csv b/dataeng/Level1/src-data/1073.csv new file mode 100644 index 00000000..0a724d4b --- /dev/null +++ b/dataeng/Level1/src-data/1073.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Myrtle, Ferro, 840830400000 diff --git a/dataeng/Level1/src-data/1073.png b/dataeng/Level1/src-data/1073.png new file mode 100644 index 00000000..0cc76579 Binary files /dev/null and b/dataeng/Level1/src-data/1073.png differ diff --git a/dataeng/Level1/src-data/1074.csv b/dataeng/Level1/src-data/1074.csv new file mode 100644 index 00000000..ed5181b9 --- /dev/null +++ b/dataeng/Level1/src-data/1074.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Gisela, Patterson, 464990400000 diff --git a/dataeng/Level1/src-data/1074.png b/dataeng/Level1/src-data/1074.png new file mode 100644 index 00000000..699a67f9 Binary files /dev/null and b/dataeng/Level1/src-data/1074.png differ diff --git a/dataeng/Level1/src-data/1075.csv b/dataeng/Level1/src-data/1075.csv new file mode 100644 index 00000000..ce375b10 --- /dev/null +++ b/dataeng/Level1/src-data/1075.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Stacey, Bishop, 493848000000 diff --git a/dataeng/Level1/src-data/1075.png b/dataeng/Level1/src-data/1075.png new file mode 100644 index 00000000..257673f9 Binary files /dev/null and b/dataeng/Level1/src-data/1075.png differ diff --git a/dataeng/Level1/src-data/1076.csv b/dataeng/Level1/src-data/1076.csv new file mode 100644 index 00000000..11f2b69c --- /dev/null +++ b/dataeng/Level1/src-data/1076.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Ray, Goins, 933451200000 diff --git a/dataeng/Level1/src-data/1076.png b/dataeng/Level1/src-data/1076.png new file mode 100644 index 00000000..ae81bf0d Binary files /dev/null and b/dataeng/Level1/src-data/1076.png differ diff --git a/dataeng/Level1/src-data/1077.csv b/dataeng/Level1/src-data/1077.csv new file mode 100644 index 00000000..474466f1 --- /dev/null +++ b/dataeng/Level1/src-data/1077.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Blake, Dickerson, 508107600000 diff --git a/dataeng/Level1/src-data/1077.png b/dataeng/Level1/src-data/1077.png new file mode 100644 index 00000000..2f9349c4 Binary files /dev/null and b/dataeng/Level1/src-data/1077.png differ diff --git a/dataeng/Level1/src-data/1078.csv b/dataeng/Level1/src-data/1078.csv new file mode 100644 index 00000000..86e493e3 --- /dev/null +++ b/dataeng/Level1/src-data/1078.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +David, May, 601074000000 diff --git a/dataeng/Level1/src-data/1078.png b/dataeng/Level1/src-data/1078.png new file mode 100644 index 00000000..db34125b Binary files /dev/null and b/dataeng/Level1/src-data/1078.png differ diff --git a/dataeng/Level1/src-data/1079.csv b/dataeng/Level1/src-data/1079.csv new file mode 100644 index 00000000..5944d6b4 --- /dev/null +++ b/dataeng/Level1/src-data/1079.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Cindy, Wilcox, 1021060800000 diff --git a/dataeng/Level1/src-data/1079.png b/dataeng/Level1/src-data/1079.png new file mode 100644 index 00000000..1980f10b Binary files /dev/null and b/dataeng/Level1/src-data/1079.png differ diff --git a/dataeng/Level1/src-data/1080.csv b/dataeng/Level1/src-data/1080.csv new file mode 100644 index 00000000..bd624446 --- /dev/null +++ b/dataeng/Level1/src-data/1080.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Mary, Nelson, 518644800000 diff --git a/dataeng/Level1/src-data/1080.png b/dataeng/Level1/src-data/1080.png new file mode 100644 index 00000000..8879569e Binary files /dev/null and b/dataeng/Level1/src-data/1080.png differ diff --git a/dataeng/Level1/src-data/1081.csv b/dataeng/Level1/src-data/1081.csv new file mode 100644 index 00000000..06561de8 --- /dev/null +++ b/dataeng/Level1/src-data/1081.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Amy, Smith, 785710800000 diff --git a/dataeng/Level1/src-data/1081.png b/dataeng/Level1/src-data/1081.png new file mode 100644 index 00000000..bd3abae2 Binary files /dev/null and b/dataeng/Level1/src-data/1081.png differ diff --git a/dataeng/Level1/src-data/1082.csv b/dataeng/Level1/src-data/1082.csv new file mode 100644 index 00000000..586ff65e --- /dev/null +++ b/dataeng/Level1/src-data/1082.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Joseph, Denton, 463089600000 diff --git a/dataeng/Level1/src-data/1082.png b/dataeng/Level1/src-data/1082.png new file mode 100644 index 00000000..eb1bb6be Binary files /dev/null and b/dataeng/Level1/src-data/1082.png differ diff --git a/dataeng/Level1/src-data/1083.csv b/dataeng/Level1/src-data/1083.csv new file mode 100644 index 00000000..543aa229 --- /dev/null +++ b/dataeng/Level1/src-data/1083.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Keith, Wiggin, 461707200000 diff --git a/dataeng/Level1/src-data/1083.png b/dataeng/Level1/src-data/1083.png new file mode 100644 index 00000000..606c64b3 Binary files /dev/null and b/dataeng/Level1/src-data/1083.png differ diff --git a/dataeng/Level1/src-data/1084.csv b/dataeng/Level1/src-data/1084.csv new file mode 100644 index 00000000..21b41e9f --- /dev/null +++ b/dataeng/Level1/src-data/1084.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Jody, Hochmuth, 752446800000 diff --git a/dataeng/Level1/src-data/1084.png b/dataeng/Level1/src-data/1084.png new file mode 100644 index 00000000..8bebb674 Binary files /dev/null and b/dataeng/Level1/src-data/1084.png differ diff --git a/dataeng/Level1/src-data/1085.csv b/dataeng/Level1/src-data/1085.csv new file mode 100644 index 00000000..98e80557 --- /dev/null +++ b/dataeng/Level1/src-data/1085.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Beverly, Livesey, 739224000000 diff --git a/dataeng/Level1/src-data/1085.png b/dataeng/Level1/src-data/1085.png new file mode 100644 index 00000000..ed69ef3d Binary files /dev/null and b/dataeng/Level1/src-data/1085.png differ diff --git a/dataeng/Level1/src-data/1086.csv b/dataeng/Level1/src-data/1086.csv new file mode 100644 index 00000000..091faace --- /dev/null +++ b/dataeng/Level1/src-data/1086.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Nicole, Pharr, 884552400000 diff --git a/dataeng/Level1/src-data/1086.png b/dataeng/Level1/src-data/1086.png new file mode 100644 index 00000000..8c02b1d6 Binary files /dev/null and b/dataeng/Level1/src-data/1086.png differ diff --git a/dataeng/Level1/src-data/1087.csv b/dataeng/Level1/src-data/1087.csv new file mode 100644 index 00000000..6a06b574 --- /dev/null +++ b/dataeng/Level1/src-data/1087.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Aaron, Fox, 1090094400000 diff --git a/dataeng/Level1/src-data/1087.png b/dataeng/Level1/src-data/1087.png new file mode 100644 index 00000000..2428c760 Binary files /dev/null and b/dataeng/Level1/src-data/1087.png differ diff --git a/dataeng/Level1/src-data/1088.csv b/dataeng/Level1/src-data/1088.csv new file mode 100644 index 00000000..10783c4d --- /dev/null +++ b/dataeng/Level1/src-data/1088.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Rachel, Robinson, 528148800000 diff --git a/dataeng/Level1/src-data/1088.png b/dataeng/Level1/src-data/1088.png new file mode 100644 index 00000000..361e7508 Binary files /dev/null and b/dataeng/Level1/src-data/1088.png differ diff --git a/dataeng/Level1/src-data/1089.csv b/dataeng/Level1/src-data/1089.csv new file mode 100644 index 00000000..13b0443a --- /dev/null +++ b/dataeng/Level1/src-data/1089.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Leah, Costales, 1072472400000 diff --git a/dataeng/Level1/src-data/1089.png b/dataeng/Level1/src-data/1089.png new file mode 100644 index 00000000..567a81c9 Binary files /dev/null and b/dataeng/Level1/src-data/1089.png differ diff --git a/dataeng/Level1/src-data/1090.csv b/dataeng/Level1/src-data/1090.csv new file mode 100644 index 00000000..d0fa66d8 --- /dev/null +++ b/dataeng/Level1/src-data/1090.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Thomas, Williams, 946501200000 diff --git a/dataeng/Level1/src-data/1090.png b/dataeng/Level1/src-data/1090.png new file mode 100644 index 00000000..2ac768a8 Binary files /dev/null and b/dataeng/Level1/src-data/1090.png differ diff --git a/dataeng/Level1/src-data/1091.csv b/dataeng/Level1/src-data/1091.csv new file mode 100644 index 00000000..9b190aeb --- /dev/null +++ b/dataeng/Level1/src-data/1091.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Opal, Morse, 463780800000 diff --git a/dataeng/Level1/src-data/1091.png b/dataeng/Level1/src-data/1091.png new file mode 100644 index 00000000..4e83071c Binary files /dev/null and b/dataeng/Level1/src-data/1091.png differ diff --git a/dataeng/Level1/src-data/1092.csv b/dataeng/Level1/src-data/1092.csv new file mode 100644 index 00000000..2985731b --- /dev/null +++ b/dataeng/Level1/src-data/1092.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Alan, Mcgraw, 469054800000 diff --git a/dataeng/Level1/src-data/1092.png b/dataeng/Level1/src-data/1092.png new file mode 100644 index 00000000..48e73678 Binary files /dev/null and b/dataeng/Level1/src-data/1092.png differ diff --git a/dataeng/Level1/src-data/1093.csv b/dataeng/Level1/src-data/1093.csv new file mode 100644 index 00000000..30939c9c --- /dev/null +++ b/dataeng/Level1/src-data/1093.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Aaron, Towe, 877809600000 diff --git a/dataeng/Level1/src-data/1093.png b/dataeng/Level1/src-data/1093.png new file mode 100644 index 00000000..a78917ad Binary files /dev/null and b/dataeng/Level1/src-data/1093.png differ diff --git a/dataeng/Level1/src-data/1094.csv b/dataeng/Level1/src-data/1094.csv new file mode 100644 index 00000000..20444099 --- /dev/null +++ b/dataeng/Level1/src-data/1094.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Cecil, Struck, 416437200000 diff --git a/dataeng/Level1/src-data/1094.png b/dataeng/Level1/src-data/1094.png new file mode 100644 index 00000000..8ba8acba Binary files /dev/null and b/dataeng/Level1/src-data/1094.png differ diff --git a/dataeng/Level1/src-data/1095.csv b/dataeng/Level1/src-data/1095.csv new file mode 100644 index 00000000..cb83462a --- /dev/null +++ b/dataeng/Level1/src-data/1095.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Corey, Davis, 1054411200000 diff --git a/dataeng/Level1/src-data/1095.png b/dataeng/Level1/src-data/1095.png new file mode 100644 index 00000000..adc7f76b Binary files /dev/null and b/dataeng/Level1/src-data/1095.png differ diff --git a/dataeng/Level1/src-data/1096.csv b/dataeng/Level1/src-data/1096.csv new file mode 100644 index 00000000..0a34f25d --- /dev/null +++ b/dataeng/Level1/src-data/1096.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +James, Peacock, 503442000000 diff --git a/dataeng/Level1/src-data/1096.png b/dataeng/Level1/src-data/1096.png new file mode 100644 index 00000000..36ca11a2 Binary files /dev/null and b/dataeng/Level1/src-data/1096.png differ diff --git a/dataeng/Level1/src-data/1097.csv b/dataeng/Level1/src-data/1097.csv new file mode 100644 index 00000000..c6b2fecd --- /dev/null +++ b/dataeng/Level1/src-data/1097.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +John, Bausch, 1013029200000 diff --git a/dataeng/Level1/src-data/1097.png b/dataeng/Level1/src-data/1097.png new file mode 100644 index 00000000..200c031f Binary files /dev/null and b/dataeng/Level1/src-data/1097.png differ diff --git a/dataeng/Level1/src-data/1098.csv b/dataeng/Level1/src-data/1098.csv new file mode 100644 index 00000000..97eac619 --- /dev/null +++ b/dataeng/Level1/src-data/1098.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Maria, Horton, 702504000000 diff --git a/dataeng/Level1/src-data/1098.png b/dataeng/Level1/src-data/1098.png new file mode 100644 index 00000000..75d0b1b7 Binary files /dev/null and b/dataeng/Level1/src-data/1098.png differ diff --git a/dataeng/Level1/src-data/1099.csv b/dataeng/Level1/src-data/1099.csv new file mode 100644 index 00000000..101cc8b3 --- /dev/null +++ b/dataeng/Level1/src-data/1099.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Samuel, Ladner, 815259600000 diff --git a/dataeng/Level1/src-data/1099.png b/dataeng/Level1/src-data/1099.png new file mode 100644 index 00000000..96d88530 Binary files /dev/null and b/dataeng/Level1/src-data/1099.png differ diff --git a/dataeng/Level1/unittests/demo-data/1000.csv b/dataeng/Level1/unittests/demo-data/1000.csv new file mode 100644 index 00000000..afd57739 --- /dev/null +++ b/dataeng/Level1/unittests/demo-data/1000.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Susan, Lee, 612302400000 diff --git a/dataeng/Level1/unittests/demo-data/1000.png b/dataeng/Level1/unittests/demo-data/1000.png new file mode 100644 index 00000000..b319625e Binary files /dev/null and b/dataeng/Level1/unittests/demo-data/1000.png differ diff --git a/dataeng/Level1/unittests/demo-data/1001.csv b/dataeng/Level1/unittests/demo-data/1001.csv new file mode 100644 index 00000000..d6c5c10c --- /dev/null +++ b/dataeng/Level1/unittests/demo-data/1001.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Rosa, Garcia, 670626000000 diff --git a/dataeng/Level1/unittests/demo-data/1001.png b/dataeng/Level1/unittests/demo-data/1001.png new file mode 100644 index 00000000..0a6f7bf0 Binary files /dev/null and b/dataeng/Level1/unittests/demo-data/1001.png differ diff --git a/dataeng/Level1/unittests/demo-data/1002.csv b/dataeng/Level1/unittests/demo-data/1002.csv new file mode 100644 index 00000000..f74740fe --- /dev/null +++ b/dataeng/Level1/unittests/demo-data/1002.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Nathan, Emery, 697237200000 diff --git a/dataeng/Level1/unittests/demo-data/1002.png b/dataeng/Level1/unittests/demo-data/1002.png new file mode 100644 index 00000000..6db6a785 Binary files /dev/null and b/dataeng/Level1/unittests/demo-data/1002.png differ diff --git a/dataeng/Level1/unittests/demo-data/1003.csv b/dataeng/Level1/unittests/demo-data/1003.csv new file mode 100644 index 00000000..da0e51ea --- /dev/null +++ b/dataeng/Level1/unittests/demo-data/1003.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Vernon, Evans, 633733200000 diff --git a/dataeng/Level1/unittests/demo-data/1003.png b/dataeng/Level1/unittests/demo-data/1003.png new file mode 100644 index 00000000..9a2eae48 Binary files /dev/null and b/dataeng/Level1/unittests/demo-data/1003.png differ diff --git a/dataeng/Level1/unittests/demo-data/1004.csv b/dataeng/Level1/unittests/demo-data/1004.csv new file mode 100644 index 00000000..f32edb85 --- /dev/null +++ b/dataeng/Level1/unittests/demo-data/1004.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Maria, Crooks, 1051214400000 diff --git a/dataeng/Level1/unittests/demo-data/1005.csv b/dataeng/Level1/unittests/demo-data/1005.csv new file mode 100644 index 00000000..38a14b90 --- /dev/null +++ b/dataeng/Level1/unittests/demo-data/1005.csv @@ -0,0 +1,2 @@ +first_name, last_name, birthts +Sharon, Lang, 700606800000 diff --git a/dataeng/Level1/unittests/demo-data/1005.png b/dataeng/Level1/unittests/demo-data/1005.png new file mode 100644 index 00000000..0ad02751 Binary files /dev/null and b/dataeng/Level1/unittests/demo-data/1005.png differ diff --git a/dataeng/Level1/unittests/demo-data/1006.png b/dataeng/Level1/unittests/demo-data/1006.png new file mode 100644 index 00000000..b75b18fd Binary files /dev/null and b/dataeng/Level1/unittests/demo-data/1006.png differ diff --git a/dataeng/Level1/unittests/demo-output/output.csv b/dataeng/Level1/unittests/demo-output/output.csv new file mode 100644 index 00000000..bbd3a953 --- /dev/null +++ b/dataeng/Level1/unittests/demo-output/output.csv @@ -0,0 +1,7 @@ +user_id,first_name,last_name,birthts,img_path +1000,Susan,Lee,612302400000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/unittests/demo-data/1000.png +1001,Rosa,Garcia,670626000000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/unittests/demo-data/1001.png +1002,Nathan,Emery,697237200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/unittests/demo-data/1002.png +1003,Vernon,Evans,633733200000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/unittests/demo-data/1003.png +1004,Maria,Crooks,1051214400000,None +1005,Sharon,Lang,700606800000,/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/unittests/demo-data/1005.png diff --git a/dataeng/Level1/unittests/test_image_path_finder.py b/dataeng/Level1/unittests/test_image_path_finder.py new file mode 100644 index 00000000..bd74b233 --- /dev/null +++ b/dataeng/Level1/unittests/test_image_path_finder.py @@ -0,0 +1,12 @@ +import unittest +import sys +sys.path.append("/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/") +from Level1.image_path_finder import process +import util + + +class TestImagePathFinder(unittest.TestCase): + + def test_output_validity(self): + process(util.input_path_test_data, util.output_path_test_data) + self.assertEqual(util.get_csv(util.output_path_test_data + "/output.csv"), [['user_id', 'first_name', 'last_name', 'birthts', 'img_path'], ['1000', 'Susan', 'Lee', '612302400000', '/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/unittests/demo-data/1000.png'], ['1001', 'Rosa', 'Garcia', '670626000000', '/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/unittests/demo-data/1001.png'], ['1002', 'Nathan', 'Emery', '697237200000', '/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/unittests/demo-data/1002.png'], ['1003', 'Vernon', 'Evans', '633733200000', '/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/unittests/demo-data/1003.png'], ['1004', 'Maria', 'Crooks', '1051214400000', 'None'], ['1005', 'Sharon', 'Lang', '700606800000', '/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/Level1/unittests/demo-data/1005.png']]) \ No newline at end of file diff --git a/dataeng/Level2/server.py b/dataeng/Level2/server.py new file mode 100644 index 00000000..36abf20b --- /dev/null +++ b/dataeng/Level2/server.py @@ -0,0 +1,87 @@ +from flask import Flask, request, jsonify +import sys +sys.path.append("/home/ehsan2754/git_workspace/provectus task/provectus-internship-task/") +from Level1.image_path_finder import process +from util import * +from apscheduler.schedulers.background import BackgroundScheduler +import atexit +import util +from typing import Union + +input_path = util.input_path_real_data +output_path = util.output_path_real_data + + +def sched_process(): + """ + function for the scheduler to make automatic updates to entries. + :return: Nothing. + """ + print("Running automated update...") + process(input_path, output_path) + print("Finished...") + + +sched = BackgroundScheduler() +sched.add_job(sched_process, 'interval', seconds=60) +sched.start() +atexit.register(lambda: sched.shutdown()) + +app = Flask(__name__) + + +@app.route('/data', methods=['POST']) +def process_data(): + """ + Forces a data update. + :return: a tuple, the number of added or updated entries, and a sorted list of the user ids for + new or updated entries + """ + return jsonify({'result': process(input_path, output_path)}) + + +def get_data(image_filter: Union[None, bool], min_age_filter: float, max_age_filter: float) -> list: + """ + Gets the data from output.csv and removes the rows that don't pass the filters. + :param image_filter: True, False, or None. Returns entries with photo path if true. Returns entries without + photo path if false. Ignores if None. + :param min_age_filter: Either -1 or a positive float. If -1 ignore, else entries' age must be higher + than min_age_filter. + :param max_age_filter: Either -1 or a positive float. If -1 ignore, else entries' age must be lower + than min_age_filter. + :return: list with filtered rows. + """ + lines = get_csv(output_path + "/output.csv") + lines = lines[1:] + ans = [] + for i in range(len(lines)): + if not check_image(lines[i], image_filter) or not check_min_age(lines[i], min_age_filter) or \ + not check_max_age(lines[i], max_age_filter): + continue + ans.append(lines[i]) + return ans + + +@app.route('/data', methods=['GET']) +def process_request(): + """ + Takes parameters from url as specified in the readme file. Queries the output.csv and returns all of the rows in the + file (except the header). + :return: a JSON file of all the rows that pass the filtering. + """ + image_filter = request.args.get('is_image_exists', default="None", type=str) + min_age_filter = request.args.get('min_age', default=-1.0, type=float) + max_age_filter = request.args.get('max_age', default=-1.0, type=float) + if min_age_filter > max_age_filter != -1: + return "min age is bigger than max age", 400 + if image_filter.lower() == "true": + image_filter = True + elif image_filter.lower() == "false": + image_filter = False + else: + image_filter = None + return jsonify({'result': get_data(image_filter, min_age_filter, max_age_filter)}) + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/dataeng/README.md b/dataeng/README.md index 3df8bdd1..9a0f1913 100644 --- a/dataeng/README.md +++ b/dataeng/README.md @@ -1,3 +1,4 @@ +# Check out my [Solution](solution.md) ### Prerequisites * Python 3.7 or greater * Docker 19.03 or greater diff --git a/dataeng/app.py b/dataeng/app.py new file mode 100644 index 00000000..99bff460 --- /dev/null +++ b/dataeng/app.py @@ -0,0 +1,83 @@ +from flask import Flask, request, jsonify +from image_path_finder import process +from util import * +from apscheduler.schedulers.background import BackgroundScheduler +import atexit +from typing import Union +import db_handler + + +minio_client = None +input_bucket = None +output_bucket = None + + +def sched_process(): + """ + function for the scheduler to make automatic updates to entries. + :return: Nothing. + """ + print("Running automated update...") + process(minio_client, input_bucket, output_bucket) + print("Finished...") + + +sched = BackgroundScheduler() +sched.add_job(sched_process, 'interval', seconds=120) +sched.start() +atexit.register(lambda: sched.shutdown()) + +app = Flask(__name__) + + +@app.route('/data', methods=['POST']) +def process_data(): + """ + Forces a data update. + :return: a tuple, the number of added or updated entries, and a sorted list of the user ids for + new or updated entries + """ + process(minio_client, input_bucket, output_bucket) + return jsonify({"Result": "Done"}) + + +def get_data(image_filter: Union[None, bool], min_age_filter: float, max_age_filter: float) -> list: + """ + Gets the data from output.csv and removes the rows that don't pass the filters. + :param image_filter: True, False, or None. Returns entries with photo path if true. Returns entries without + photo path if false. Ignores if None. + :param min_age_filter: Either -1 or a positive float. If -1 ignore, else entries' age must be higher + than min_age_filter. + :param max_age_filter: Either -1 or a positive float. If -1 ignore, else entries' age must be lower + than min_age_filter. + :return: list with filtered rows. + """ + lines = db_handler.get_users_data('users') + ans = [] + for i in range(len(lines)): + if not check_image(lines[i], image_filter) or not check_min_age(lines[i], min_age_filter) or \ + not check_max_age(lines[i], max_age_filter): + continue + ans.append(lines[i]) + return ans + + +@app.route('/data', methods=['GET']) +def process_request(): + """ + Takes parameters from url as specified in the readme file. Queries the output.csv and returns all of the rows in the + file (except the header). + :return: a JSON file of all the rows that pass the filtering. + """ + image_filter = request.args.get('is_image_exists', default="None", type=str) + min_age_filter = request.args.get('min_age', default=-1.0, type=float) + max_age_filter = request.args.get('max_age', default=-1.0, type=float) + if min_age_filter > max_age_filter != -1: + return "min age is bigger than max age", 400 + if image_filter.lower() == "true": + image_filter = True + elif image_filter.lower() == "false": + image_filter = False + else: + image_filter = None + return jsonify({'result': get_data(image_filter, min_age_filter, max_age_filter)}) diff --git a/dataeng/config.py b/dataeng/config.py new file mode 100644 index 00000000..39f4aa64 --- /dev/null +++ b/dataeng/config.py @@ -0,0 +1,7 @@ +access_key = "minio-access-key" +secret_key = "minio-secret-key" +db_name = "internship" +user = "postgres" +password = "postgres" +db_host = "db" +minio_host = "minio" diff --git a/dataeng/db_handler.py b/dataeng/db_handler.py new file mode 100644 index 00000000..a6a3e20a --- /dev/null +++ b/dataeng/db_handler.py @@ -0,0 +1,108 @@ +import psycopg2 +import config + + +def init(): + """ + Initiates the connection with the database 'dbname' + :return: (conn, crsr) -> (the connection object, the cursor to perform commands) + """ + conn = psycopg2.connect(dbname=config.db_name, user=config.user, + password=config.password, host=config.db_host) + return conn, conn.cursor() + + +def get_users_data(table_name): + """ + Retrieves all the data in the table table_name + :param table_name: the name of the table + :return: a list that contains all the data in the table + + NOTE: + the returned list will contain all the information about the users right after each other + without having each user in a separate list. + For example: returned_list = [user_id1, first_name1, last_name1, birthdate1, img_path1, user_id2, first_name2, .....] + """ + conn, crsr = init() + + crsr.execute(f"SELECT user_id, first_name, last_name, birthdate, img_path FROM {table_name};") + data = crsr.fetchall() + + conn.commit() + crsr.close() + conn.close() + + res = [] + for row in data: + res.append(row) + return res + + +def get_ids(table_name): + """ + Retrieves all the ids of the users inside the 'table_name' table + :param table_name: the name of the table + :return: a list of the ids of the users inside the table + """ + conn, crsr = init() + + crsr.execute(f"SELECT user_id FROM {table_name};") + ids = crsr.fetchall() + + conn.commit() + conn.close() + crsr.close() + + return [id[0] for id in ids] + + +def update_row(table_name, user): + """ + + :param user: [first_name, last_name, birthts, img_path, user_id] + :return: + """ + conn, crsr = init() + + user = [user[1], user[2], user[3], user[4], user[0]] + + crsr.execute(f""" + UPDATE {table_name} SET first_name = %s, last_name = %s, + birthdate = %s, img_path = %s WHERE user_id = %s; + """, user) + + conn.commit() + conn.close() + + +def insert_row(table_name, user): + conn, crsr = init() + + crsr.execute( + f"INSERT INTO {table_name} (user_id, first_name, last_name, birthdate, img_path) " + f"VALUES (%s, %s, %s, %s, %s)", user) + + conn.commit() + conn.close() + + +def create_table_users(): + """ + Creates the table users that we will use to migrate our data from output.csv to. + """ + conn, crsr = init() + + crsr.execute(""" + CREATE TABLE IF NOT EXISTS users( + id SERIAL PRIMARY KEY NOT NULL, + user_id varchar (20) NOT NULL, + first_name varchar (30) NOT NULL, + last_name varchar (30) NOT NULL, + birthdate varchar (30) NOT NULL, + img_path varchar (250) NOT NULL + ); + """) + + conn.commit() + crsr.close() + conn.close() diff --git a/dataeng/docker-compose.yml b/dataeng/docker-compose.yml new file mode 100644 index 00000000..0180622a --- /dev/null +++ b/dataeng/docker-compose.yml @@ -0,0 +1,57 @@ +version: '3' +services: + minio: + image: bitnami/minio:latest + restart: always + environment: + - MINIO_ACCESS_KEY=minio-access-key + - MINIO_SECRET_KEY=minio-secret-key + volumes: + - ./minio:/data + ports: + - 9000:9000 + - 9001:9001 + + minio-create-bucket: + image: minio/mc + depends_on: + - minio + entrypoint: > + /bin/sh -c " + /usr/bin/mc config host add myminio http://minio:9000 minio-access-key minio-secret-key; + /usr/bin/mc mb myminio/datalake; + /usr/bin/mc policy download myminio/datalake; + exit 0; + " + db: + image: postgres:13.3 + restart: always + environment: + POSTGRES_DB: internship + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + volumes: + - ./postgres-data:/var/lib/postgresql + ports: + - 5432:5432 + + pgadmin: + image: chorss/docker-pgadmin4 + restart: always + volumes: + - ./pgadmin:/data + ports: + - 5050:5050 + depends_on: + - db + + web: + build: . + ports: + - 5000:5000 + depends_on: + - db + - pgadmin + - minio + - minio-create-bucket + diff --git a/dataeng/dockerfile b/dataeng/dockerfile new file mode 100644 index 00000000..400b2a6c --- /dev/null +++ b/dataeng/dockerfile @@ -0,0 +1,9 @@ +FROM python:3.7.12-bullseye +WORKDIR /code +ENV FLASK_APP=app.py +ENV FLASK_RUN_HOST=0.0.0.0 +COPY requirements.txt requirements.txt +RUN pip3 install -r requirements.txt +EXPOSE 5000 +COPY . . +CMD ["python3", "main.py"] diff --git a/dataeng/image_path_finder.py b/dataeng/image_path_finder.py new file mode 100644 index 00000000..d5fa79da --- /dev/null +++ b/dataeng/image_path_finder.py @@ -0,0 +1,77 @@ +from util import * +from minio import Minio +import os +import io +import db_handler + + +def append_to_csv(minio_client: Minio, input_bucket: str, object_name: str, user_id: (bool, int), output_bucket: str) -> None: + """ + Given the path to the .csv file and the user id, create the data needed and then + append it to output_file/output.csv. + :param output_file: path to where the output file should be. + :param user_id: user id to be added to the output file. + :param path_to_csv: path to the .csv file. + :param user_png: is there a photo for this .csv file. + :return: true if the output.csv was changed, false if not. + """ + lines = get_csv(minio_client, output_bucket, "output.csv", 5) + if user_id[0]: + png_path = input_bucket + "/" + object_name[:-4] + png_path += ".png" + else: + png_path = "None" + temp = get_csv(minio_client, input_bucket, object_name, 3) + line_to_write = [str(user_id[1]), *temp[1], png_path] + lines.append(line_to_write) + put_csv("temp_output.csv", lines) + minio_client.remove_object(output_bucket, "output.csv") + minio_client.fput_object(output_bucket, "output.csv", "temp_output.csv") + os.remove("temp_output.csv") + if line_to_write[0] in db_handler.get_ids('users'): + db_handler.update_row("users", line_to_write) + else: + db_handler.insert_row("users", line_to_write) + + +def find_png_and_id(minio_client: Minio, input_bucket: str, obj_name: str) -> (bool, int): + """ + Takes a path to a .csv file and checks if there's a .png file with the same name in the same directory. + if the file exists then returns the name of the file (since this is also the user id), if it doesn't exist + return -1. + :param input_path: path to the .csv file. + :return: user id if the file exists or -1 otherwise. + """ + try: + user_id = int(obj_name[:-4]) + except ValueError: + return False, -1 + png_name = obj_name[:-4] + png_name += ".png" + try: + minio_client.stat_object(input_bucket, png_name) + return True, user_id + except: + return False, user_id + + +def process(minio_client: Minio, input_bucket: str, output_bucket: str) -> None: + """ + Reads all files in input_path (absolute path). looks for a .png and a .csv files that have the same name and + combines them. Stores output in output_path/output.csv. + Returns the number of files found and their names. + :param minio_client: path used for finding the input. + :param input_bucket: path used for finding the input. + :param output_bucket: path used for finding the output file. + :return: number of files found and their names. + """ + minio_client.remove_object(output_bucket, "output.csv") + minio_client.put_object( + output_bucket, "output.csv", io.BytesIO(b"user_id,first_name,last_name,birthts,img_path"), 45, + ) + for obj in minio_client.list_objects(input_bucket): + if obj.object_name.endswith('.csv'): + user_id = find_png_and_id(minio_client, input_bucket, obj.object_name) + if user_id[1] == -1: + continue + append_to_csv(minio_client, input_bucket, obj.object_name, user_id, output_bucket) diff --git a/dataeng/main.py b/dataeng/main.py new file mode 100644 index 00000000..768cec06 --- /dev/null +++ b/dataeng/main.py @@ -0,0 +1,47 @@ +from minio import Minio +import config +import os +import io +import app +from minio.select import SelectRequest, CSVInputSerialization, CSVOutputSerialization +import db_handler + + +def get_minio_client(access, secret): + return Minio( + config.minio_host + ":9000", + access_key=access, + secret_key=secret, + secure=False + ) + + +if __name__ == "__main__": + db_handler.create_table_users() + minio_client = get_minio_client(config.access_key, config.secret_key) + if not minio_client.bucket_exists("src"): + minio_client.make_bucket("src") + if not minio_client.bucket_exists("res"): + minio_client.make_bucket("res") + + try: + minio_client.select_object_content( + "res", + "output.csv", + SelectRequest( + "select * from S3Object", + CSVInputSerialization(), + CSVOutputSerialization(), + request_progress=True, + ), + ) + except: + minio_client.put_object( + "res", "output.csv", io.BytesIO(b"user_id,first_name,last_name,birthts,img_path"), 45, + ) + + app.minio_client = minio_client + app.input_bucket = "src" + app.output_bucket = "res" + port = int(os.environ.get('PORT', 5000)) + app.app.run(host='0.0.0.0', port=port) diff --git a/dataeng/requirements.txt b/dataeng/requirements.txt new file mode 100644 index 00000000..3820d264 --- /dev/null +++ b/dataeng/requirements.txt @@ -0,0 +1,19 @@ +APScheduler==3.8.0 +backports.zoneinfo==0.2.1 +certifi==2021.5.30 +charset-normalizer==2.0.6 +click==8.0.1 +Flask==2.0.2 +idna==3.2 +itsdangerous==2.0.1 +Jinja2==3.0.2 +MarkupSafe==2.0.1 +minio==7.1.0 +psycopg2==2.9.1 +python-dateutil==2.8.2 +pytz==2021.3 +requests==2.26.0 +six==1.16.0 +tzlocal==2.0.0 +urllib3==1.26.7 +Werkzeug==2.0.2 diff --git a/dataeng/solution.md b/dataeng/solution.md new file mode 100644 index 00000000..a5504a2d --- /dev/null +++ b/dataeng/solution.md @@ -0,0 +1,239 @@ +# Provectus-Internship-Task +In this file I will explain how to run and test my solution to the test task. I will also explain some of the logic in the code. Reading this file plus the documentation in the code itself should be sufficient to understand how this code works. The solution to the theoretical questions will also be included in this file. + +## Applicant +[![Ehsans's GitHub stats](https://github-readme-stats.vercel.app/api?username=ehsan2754)](https://github.com/ehsan2754/github-readme-stats) + + +![Github]( https://img.shields.io/badge/Github-000000?style=for-the-badge&logo=github&logoColor=white) [Ehsan Shaghaei](https://github.com/Ehsan2754) + + +![Telegram]( https://img.shields.io/badge/Telegram-2CA5E0?style=for-the-badge&logo=telegram&logoColor=white) [Ehsan Shaghaei](https://t.me/ethanshagaei) + + +## Table of Contents +1. [ Data Processing ](#data) +2. [ Theoretical Questions ](#theo) + + + +# Data Processing Level 3 + +## Table of Contents +1. [ Installing The Service ](#install) +2. [ Using The Service ](#use) +3. [ Understanding The Code ](#code) + + +# 1. Installing The Service + +First clone this repo somewhere on your machine. From now on we will call the path to where you cloned the repo {installation path}. + +Then inside the repo run +``` +sudo docker-compose up --build +``` +This will run and create some folders in {installation path}. Namely it will create {installation path}/minio, {installation path}/pgadmin, and {installation path}/postgres-data. + +After it creates these folders docker-compose will keep crashing and restarting. Stop and the services with `Ctrl+C` and run: +``` +sudo docker-compose down +``` + +The docker-compose crashes since it needs access permissions to these folders but these folders are created without the permissions. To fix this problem one can do something like +``` +sudo chmod 777 minio/ +sudo chmod 777 pgadmin/ +sudo chmod 777 postgres-data/ +``` + +Now running + +``` +sudo docker-compose up +``` +Will run everything without crashing. + +If you wait some time until docker-compose runs the web service then inside the {installation path}/minio you should have 2 folders named `src` and `res`. + +When the service wants to read input data it will only look inside `src` so if you want to add input data for processing then you need to copy it inside `src` and since this folder is created without permissions to copy you might need to do + +``` +sudo chmod 777 minio/src/ +``` +before you are allowed to copy inside `src`. + +Inside `res` there will be a file called `output.csv` which is the same `output.csv` described in the the test task. + +Now you should have a running service. + + + +# 2. Using The Service + +The services need some time to set up after building. You should wait for a few seconds after docker-compose builds all services to make sure everything is working. On slow laptops it will take some time for services to run after building. + +After running docker-compose as specified in the previous step you should have multiple services running. One of them is a flask server. + +The flask server starts a scheduler that processes the input data in `src` every set amount of time (the default is 900 seconds and can be edited form `config.py` file) and puts the output data in `output.csv` and postgres DB. + +You can interact with the flask server with the following requests: + +* a POST request on http://localhost:5000/data to force it to process the data in `src` instead of waiting for the schedular. +* a GET request on http://localhost:5000/data to return the data in the postgres DB. + +The GET request can use a query string to give filters for the retrieved data. The filters are `is_image_exists`, `min_age`, and `max_age`. For example: + +* http://localhost:5000/data?is_image_exists=true&min_age=30.5 will return all users that have a photo and their age >= 30.5. +* http://localhost:5000/data?max_age=30.5 will return all users whose age <= 30.5 regardless of their photo status. +* http://localhost:5000/data?is_image_exists=false will return all users without a photo. + +To check that the data is getting to the postgres DB correctly you can use the `pgadmin` service on http://localhost:5050 with the password `postgres`. Then make a connection to the postgres service on port 5432. Note that when you are making a connection to the postgres service the hostname/address is `db` and not `localhost` since docker images have their own dns names. The username and password for the connection are `postgres`. + + +# 3. Understanding The Code + +The `main.py` file is the launching point of the web service. It starts by making a connection with the `minio` and `postgres` services. It then creates 2 buckets in `minio` if they don't exist. The names of these buckets are taken from `config.py` file. After that, `main.py` starts a flask server which can be found in `app.py`. + +The flask server starts a scheduler to process the input data periodically and has two end-points as defined in "Using The Service". Processing the input data, whether that is done because of scheduler or a call to the end-point, happens by making a call to a function called `process` in the script `image_path_finder.py` which handles the updating. All the previous scripts make use of helper functions defined in `util.py` and from environment variables defined in `config.py`. + +Also, whenever the app needs to interact with the postgres DB it will use some of the functions in `db_handler.py`. For example, when sending the output from `output.csv` to the postgres DB it will use the function `handle_row` inside `db_handler.py`. + +### Different DNS Names In The Containers + +In the dockerfile you will notice that we define an environment variable called `IS_DOCKER`. This environment variable will tell the web service if it is running from docker or not. When the web service is running it will look at that variable and depending on it, the service will either connect to `minio:9000` or `localhost:9000` for establishing the connection to `minio` service. That is because if the web service is running from inside the container then it needs to use the dns names for the containers instead of `localhost`. The same thing will happen for connecting with the database. It will either use `db` or `localhost`. + +### Running The Web Service Without Docker + +Due to handling the dns names as defined above you can actually launch the service from outside the docker without problems. + +First go you `docker-compose.yml` and comment out or delete everything related to the web service. Then run +``` +sudo docker-compose up +``` +Now you can use the service from the terminal or any IDE and it will work as expected provided that docker-composed finished building and setting up. This is helpful if you wanted to use a debugger on the code. Make sure you have all the dependencies in `requirements.txt` installed. + +### Safety Note +The `config.py` shouldn't actually be uploaded to the git repo and `dockerfile` shouldn't use explicit values in the `ENV` statement in a production environment because they contain passwords, account names, environment variables. It was done this way to ease the use of this repo since this is just a test task. + +The proper way of handling that is creating a `.env` file that defines all those variables and use it in `dockerfile` and the web service itself and gitignoring that file when pushing to the repo. + + + +# Theoretical Questions + +## Table of Contents +1. [ SQL ](#sql) +2. [ Algorithms And Data Structures ](#dsa) +3. [ Linux Shell ](#linux) + + + +# 1. SQL + + 1. +``` +SELECT users.id FROM +users LEFT JOIN departments ON users.id=departments.user_id +WHERE departments.department_id != 1 OR departments.user_id IS NULL; +``` + + 2. Here we changed the table name from `user` to `users` + + ``` + SELECT lastname FROM users + GROUP BY lastname + HAVING count(lastname) > 1 + ``` + + 3. Here we changed the table name from `user` to `users` + + ``` + SELECT username, salary FROM + (salary JOIN users ON salary.id = users.id) + ORDER BY salary DESC LIMIT 1 OFFSET 1 + ``` + + +# 2. Algorithms And Data Structures + + 1. + +``` +def count_connections(list1: list, list2: list) -> int: + ans = 0 + + count = {} + + for i in list1: + if i in count: + count[i] += 1 + else: + count[i] = 1 + + for i in list2: + if i in count: + ans += 1 + + return ans +``` + + 2. + The time complexity for this solution is O(N) since dictionaries are hashmaps and we only loop over the string and do nothing else. + + The memory complexity is O(1) since we use a constant number of variables. `last_idx` would have at max 26 values inside it, therefore, constant memory. +``` +def calc(s: str) -> int: + last_idx = {} + ans = 0 + start_idx = 0 + + for i in range(len(s)): + if s[i] in last_idx: + start_idx = max(start_idx, last_idx[s[i]] + 1) + + ans = max(ans, i - start_idx + 1) + + last_idx[s[i]] = i + + return ans +``` + 3. + ``` +def bs(nums: list, target: int) -> int: + ans = len(nums) + l = 0 + r = len(nums) - 1 + while l <= r: + mid = (l + r) // 2 + if nums[mid] == target: + return mid + elif nums[mid] > target: + ans = mid + r = mid - 1 + else: + l = mid + 1 + + return ans + ``` + + +# 3. Linux Shell + + 1. `sudo lsof -i :80 ; lsof -i :443` + + 2. `cat /proc/PID/environ | tr '\0' '\n'` + + 3. + + To run in the background: + + `nohup python /path/to/my_program.py &` + + To close first you have to find the PID using this: + + `ps ax | grep my_program.py` + + and then kill the process with: + + `kill PID` diff --git a/dataeng/util.py b/dataeng/util.py new file mode 100644 index 00000000..9bcad307 --- /dev/null +++ b/dataeng/util.py @@ -0,0 +1,99 @@ +import csv +from minio import Minio +from minio.select import SelectRequest, CSVInputSerialization, CSVOutputSerialization + + +def millisecond_to_years(x): + """ + Turns x millisecond to years. + :param x: number of milliseconds. + :return: the equivalent number of years. + """ + return x / 31556952000 + + +def check_image(row, image_filter): + """ + Checks if a certain row in the output.csv file correctly passes the filter. + :param row: an entry in the .csv file. + :param image_filter: True, False, or None. + :return: if it passes or not. + """ + if image_filter is None: + return True + if image_filter: + if row[-1] != "None": + return True + if not image_filter: + if row[-1] == "None": + return True + return False + + +def check_min_age(row, min_age_filter): + """ + Checks if a certain row in the output.csv file correctly passes the filter. + :param row: an entry in the .csv file. + :param min_age_filter: minimum age in years. + :return: if it passes or not. + """ + if min_age_filter == -1: + return True + if millisecond_to_years(int(row[-2])) >= min_age_filter: + return True + return False + + +def check_max_age(row, max_age_filter): + """ + Checks if a certain row in the output.csv file correctly passes the filter. + :param row: an entry in the .csv file. + :param max_age_filter: maximum age in years. + :return: if it passes or not. + """ + if max_age_filter == -1: + return True + if millisecond_to_years(int(row[-2])) <= max_age_filter: + return True + return False + + +def get_csv(minio_client: Minio, bucket: str, obj: str, num_col: int): + """ + Given an absolute path to a .csv, returns the contents of the csv file. + :param path: path to the .csv + :return: contents of the csv file. + """ + with minio_client.select_object_content( + bucket, + obj, + SelectRequest( + "select * from S3Object", + CSVInputSerialization(), + CSVOutputSerialization(), + request_progress=True, + ), + ) as result: + for data in result.stream(): + x = data.decode().replace("\n", ",").split(",") + for i in range(len(x)): + x[i] = x[i].strip('"').strip(" ") + y = [i for i in x if i != ""] + ans = [] + k = 0 + temp = [] + for i in y: + if not k: + ans.append(temp.copy()) + temp.clear() + temp.append(i) + k += 1 + k %= num_col + ans.append(temp.copy()) + ans = ans[1:] + return ans + + +def put_csv(path: str, lines: list): + w = csv.writer(open(path, "w+")) + w.writerows(lines)