diff --git a/.tt_skip b/.tt_skip index d03b8a1..5e9ef8d 100644 --- a/.tt_skip +++ b/.tt_skip @@ -1 +1,2 @@ -# add paths to tool repositories that should be ignored by CI \ No newline at end of file +# add paths to tool repositories that should be ignored by CI +tools/visinity \ No newline at end of file diff --git a/tools/visinity/archive_generate.py b/tools/visinity/archive_generate.py new file mode 100644 index 0000000..f74eb2a --- /dev/null +++ b/tools/visinity/archive_generate.py @@ -0,0 +1,230 @@ +import csv +import json +import os +import re +import shutil +import sys +import time + + +from zipfile import ZipFile, ZIP_BZIP2 + +from minerva_analysis import app + + +app.config['IS_DOCKER'] = False + +# create a non-serving test client to replace http request +requests = app.test_client() + +args = sys.argv[1:] + +input_dict = {} +with open(args[0], 'r') as f: + input_dict = json.load(f) + +out_file = args[1] + +name_list = [] +with open(input_dict['dataset_names'], 'r') as f: + _reader = csv.reader(f) + header = next(_reader) + assert header[0].lower() == 'names' + for row_list in _reader: + row = row_list[0] + if row in name_list: + raise ValueError(f'dataset name {row} is a duplicate!' + ' please ensure names are unique') + s = re.sub('[^A-z0-9.~]+', '-', row) + if s != row: + row = s + name_list.append(row) + +channel_files = input_dict['channel_files'] +label_files = input_dict['label_files'] + +assert len(name_list) == len(channel_files) == len(label_files) + +radius = input_dict['n_radius'] +celltype_csv = input_dict['celltype_csv'] + +# ensure env +data_path = f"{os.getcwd()}/minerva_analysis/data" +config_path = f"{data_path}/config.json" + +if not os.path.isdir(data_path): + os.mkdir(data_path) + +if not os.path.isfile(config_path): + with open(config_path, 'w') as f: + f.write('{}') + +url = 'http://127.0.0.1:8000' + +with open(f'{data_path}/names.csv', 'w') as name_csv_file: + name_csv_file.write( + ','.join(name_list) + ) + +shutil.copyfile(celltype_csv, 'celltype.csv') + +payload_data = { + "action": "Upload", + "celltype_file": open('celltype.csv', 'rb'), + "neighborhood_radius": int(radius), +} + +for i, name in enumerate(name_list): + new_quant_csv = f'quant{i}.csv' + shutil.copyfile(input_dict['quant_csvs'][i], new_quant_csv) + _sub_d = { + f'name-{i+1}': name, + f'channel_file-{i+1}': channel_files[i], + f'label_file-{i+1}': label_files[i], + f'csv_file-{i+1}': open(new_quant_csv, 'rb'), + } + payload_data.update(_sub_d) + + +# payload = { +# 'data': { +# "action": "Upload", +# "name-1": dataset_name, +# "neighborhood_radius": int(n_radius), +# "channel_file-1": channel_file, +# "label_file-1": label_file, +# "csv_file-1": open(quant_target, 'rb'), +# "celltype_file": open(celltype_target, 'rb') +# } +# } + +resp = requests.post(f"{url}/upload", data=payload_data) + +resp_regex = 'passVariablesToFrontend\((.*)\);' +json_string = re.search(resp_regex, resp.text).group(1) +combinedChannelData = json.loads(json_string) + + +# def prep_header(_pre_sorted_headers: List[dict]): +# header_list = [] +# # x_pos_terms = ['x_centroid', 'cellposition_x', 'x_position', 'x'] +# # y_pos_terms = ['y_centroid', 'cellposition_y', 'y_position', 'y'] +# # phenotype_terms = ['celltype', 'phenotype'] + +# _header_prefix = [None, None, None] +# _sorted_headers = [] +# for header in _pre_sorted_headers[1:]: +# if header['fullName'].lower() in x_pos_terms: +# assert (_header_prefix[0] is None) +# _header_prefix[0] = header +# elif header['fullName'].lower() in y_pos_terms: +# assert (_header_prefix[1] is None) +# _header_prefix[1] = header +# elif header['fullName'].lower() in phenotype_terms: +# assert (_header_prefix[2] is None) +# _header_prefix[2] = header +# else: +# _sorted_headers.append(header) +# _i = 1 +# all_headers = _header_prefix + _sorted_headers +# all_headers: dict +# for header in all_headers: +# header_list.extend( +# chunkify(header['fullName'], _i) +# ) +# return header_list + + +def chunkify(fullname, idx): + non_norm_markers = [ + 'majoraxislength', 'number_neighbors', 'minoraxislength', 'neighbor_1', + 'solidity', 'eccentricity', 'y_position', 'x_position', 'neighbor_2', + 'percent_touching', 'orientation', 'neighbor_4', 'extent', 'cellid', + 'field_col', 'eulernumber', 'neighbor_3', 'neighbor_5', 'perimeter', + 'field_row'] + + return [ + { + "name": f"fullName{idx}", + "value": fullname + }, + { + "name": f"name{idx}", + "value": fullname + }, + { + "name": f"normalize{idx}", + "value": "off" if fullname.lower() in non_norm_markers else "on" + }, + ] + + +x_pos_terms = ['x_centroid', 'cellposition_x', 'x_position', 'x'] +y_pos_terms = ['y_centroid', 'cellposition_y', 'y_position', 'y'] +phenotype_terms = ['celltype', 'phenotype'] + +# headers should all be the same, so using [0] +pre_sorted_headers = combinedChannelData[0]["csvHeader"] + +# fix issue with filename "*_" being mistakenly assumed to be +# non-segmentation files; weirdly channel names dont seem to matter, +# only if it has an underscore and number +for channel in combinedChannelData: + channel["labelName"] = 'segmentation' + +idfield = chunkify(pre_sorted_headers[0]['fullName'], 0) + +header_prefix = [None, None, None] +sorted_headers = [] + +for header in pre_sorted_headers[1:]: + if header['fullName'].lower() in x_pos_terms: + assert (header_prefix[0] is None) + header_prefix[0] = header + elif header['fullName'].lower() in y_pos_terms: + assert (header_prefix[1] is None) + header_prefix[1] = header + elif header['fullName'].lower() in phenotype_terms: + assert (header_prefix[2] is None) + header_prefix[2] = header + else: + sorted_headers.append(header) + + +all_headers = header_prefix + sorted_headers +all_headers: dict + +headerlist = [] +i = 1 +for header in all_headers: + headerlist.extend( + chunkify(header['fullName'], i) + ) + i += 1 + +payload = { + 'originalData': combinedChannelData, + 'headerList': headerlist, + 'normalizeCsv': False, + 'idField': idfield +} + +# execute request +start = time.time() +resp = requests.post(f"{url}/save_config", json=payload) +duration = time.time() - start +print(f"processing duration:\n\t{duration} sec ({duration/60} mins)") + +time.sleep(5) + +# extract relevant config.json bit +with open(config_path, 'r') as innie: + config = json.load(innie) + +# zip it n ship it +with ZipFile(f'{out_file}', 'w', ZIP_BZIP2) as zipped: + path_len = len(data_path) + 1 + for base, dirs, files in os.walk(data_path): + for file in files: + file_name = os.path.join(base, file) + zipped.write(file_name, file_name[path_len:]) diff --git a/tools/visinity/archive_ingest.py b/tools/visinity/archive_ingest.py new file mode 100644 index 0000000..4ac5817 --- /dev/null +++ b/tools/visinity/archive_ingest.py @@ -0,0 +1,48 @@ +import json +import sys + +from os.path import isfile + +from zipfile import ZipFile + + +with open(sys.argv[1], 'r') as f: + arg_dict = json.load(f) + +arch_zip = arg_dict['vis_archive'] +og_channels = arg_dict['channel_files'] +og_segs = arg_dict['label_files'] + + +data_path = '/visinity/minerva_analysis/data' +conf_path = f'{data_path}/config.json' + + +with ZipFile(arch_zip, 'r') as zippy: + zippy.extractall(f'{data_path}') + +with open(f'{data_path}/names.csv', 'r') as f: + name_list = f.read().split(',') + +with open(conf_path, 'r') as f: + config = json.load(f) + + +def reconfigure(channel, seg, old_config): + subconfig = dict(old_config) + if not subconfig['segmentation'].startswith(data_path) or not isfile(subconfig['segmentation']): + subconfig['segmentation'] = seg + subconfig['channelFile'] = channel + return subconfig + + +new_config = {} + +for i, name in enumerate(name_list): + name = name_list[i] + channel = og_channels[i] + seg = og_segs[i] + new_config[name] = reconfigure(channel, seg, config[name]) + +with open(conf_path, 'w') as f: + f.write(json.dumps(new_config)) diff --git a/tools/visinity/dependencies/Dockerfile b/tools/visinity/dependencies/Dockerfile new file mode 100644 index 0000000..749fed6 --- /dev/null +++ b/tools/visinity/dependencies/Dockerfile @@ -0,0 +1,49 @@ +FROM mambaorg/micromamba + +ARG TAG +ARG PIP_NO_COMPILE="true" + +USER root + +RUN <> /etc/pip.conf + chmod -R 0777 /visinity + chown -R visinity /visinity + micromamba install -n base -y -c conda-forge python=3.9 pip=24.0 openslide-python=1.3.1 + micromamba install -n base -y -f /visinity/requirements.yml + micromamba clean --all --force-pkgs-dirs -y +VISINITY + +RUN <> /bin/vis-env + chmod 755 /bin/vis-env +ENVMAKER + +USER visinity + +SHELL [ "micromamba", "run", "-n", "base", "/bin/bash", "-c" ] + +RUN < + + + + quay.io/goeckslab/visinity:1.17v0 + + + + echo @TOOL_VERSION@ + + + + 10.1109/TVCG.2022.3209378 + + + + 1.17 + 0 + \ No newline at end of file diff --git a/tools/visinity/static/images/dataset_names_example.png b/tools/visinity/static/images/dataset_names_example.png new file mode 100644 index 0000000..b403a13 Binary files /dev/null and b/tools/visinity/static/images/dataset_names_example.png differ diff --git a/tools/visinity/static/images/phenotype_id_example.png b/tools/visinity/static/images/phenotype_id_example.png new file mode 100644 index 0000000..ad179c8 Binary files /dev/null and b/tools/visinity/static/images/phenotype_id_example.png differ diff --git a/tools/visinity/test-data/celltypes.csv b/tools/visinity/test-data/celltypes.csv new file mode 100644 index 0000000..bdb9562 --- /dev/null +++ b/tools/visinity/test-data/celltypes.csv @@ -0,0 +1,5 @@ +phenotypeID,phenotype +0,epithelial +1,tcell +2,bcell +3,macrophage diff --git a/tools/visinity/test-data/dataset_names.csv b/tools/visinity/test-data/dataset_names.csv new file mode 100644 index 0000000..0409edb --- /dev/null +++ b/tools/visinity/test-data/dataset_names.csv @@ -0,0 +1,2 @@ +names +img_1 diff --git a/tools/visinity/test-data/test_image.ome.tiff b/tools/visinity/test-data/test_image.ome.tiff new file mode 100644 index 0000000..e4a8fd5 Binary files /dev/null and b/tools/visinity/test-data/test_image.ome.tiff differ diff --git a/tools/visinity/test-data/test_mask.tiff b/tools/visinity/test-data/test_mask.tiff new file mode 100644 index 0000000..b834911 Binary files /dev/null and b/tools/visinity/test-data/test_mask.tiff differ diff --git a/tools/visinity/test-data/test_quant.csv b/tools/visinity/test-data/test_quant.csv new file mode 100644 index 0000000..d764d38 --- /dev/null +++ b/tools/visinity/test-data/test_quant.csv @@ -0,0 +1,41 @@ +CellID,CHAN_0,CHAN_1,X_centroid,Y_centroid,Area,MajorAxisLength,MinorAxisLength,Eccentricity,Solidity,Extent,Orientation,phenotype +1,51.14091983548991,2.290488748001694,1275.204120950442,1094.3768565455614,146374,2281.259866065922,1725.3575809195545,0.6542043213246477,0.051162046,0.047094427,-0.168143532,0 +2,72.73037051491572,2.271715003536844,1269.4086518525746,1041.6905871161305,81994,2340.824450942512,1704.9999369166883,0.6851778308365126,0.028878245,0.026567878769026585,-0.117566742,0 +3,91.61613958560524,2.275373218516151,1281.3469334035271,891.7220321137141,53186,2154.458074124651,1655.686514736985,0.6398578788941125,0.019197797303082346,0.017920753247138328,-0.163770793,0 +4,109.59162831583846,2.3391850664460416,1251.6879621490252,781.7976368995295,38678,1716.3086063256324,1588.212399104097,0.3790766322661075,0.014503796756351365,0.013370141147319749,-0.091729022,0 +5,121.25788747508815,2.433726812816189,1282.6231795186263,725.4632224436609,32615,1562.8748336066526,1179.9250437275666,0.6557582763139613,0.012590399,0.011647681820259645,-1.52676913,0 +6,127.28789663068368,2.3890742558063462,1379.1208374223095,721.4061171082761,30570,1574.5541361457574,727.3975018,0.8868953107270026,0.011852751373709457,0.011250879147450555,-1.546165455,0 +7,131.83924437137202,2.4019394020736167,1406.1298862530618,753.6007784,29803,1417.6436167021993,465.42945730406717,0.9445693247449122,0.012125929,0.011052959,-1.555221037,0 +8,128.73662078136795,2.5135679114401754,1447.0016892499011,795.7843151349603,27823,1370.3318612124478,272.8423027256872,0.9799778099964441,0.012236252961430053,0.011170078,-1.567725502,0 +9,133.39438356647054,2.3702124660864663,1479.073147528276,839.3319474296184,28381,1263.4384219615058,202.11057549754824,0.9871220905763974,0.014669184839688723,0.012526775,1.5648621719026883,0 +10,131.94994053812994,2.509179426192954,1487.9753604875873,882.2431247212725,26908,1287.4395591514278,191.7685058180412,0.9888442158381593,0.017671021376892256,0.012839464701015305,1.5703493948332217,0 +11,138.00660611065234,2.6541550934614517,1538.203025298401,924.5869679453494,26642,1338.4843733514879,205.88022621561413,0.9880995182397931,0.022169816,0.018014363077114483,-1.565469118,0 +12,138.93794589774078,2.552318668252081,1508.5394247919144,967.5520585612367,26912,1327.2062938575607,236.75074077450188,0.9839611708959591,0.023415686735142237,0.020446895964868293,-1.568727705,0 +13,141.9596202,2.4521467246244186,1545.1672386181651,1013.1343704720506,26226,1447.2073417270547,268.8997615672153,0.9825864552886368,0.018350344217640428,0.016538442135001514,1.5661073556608107,1 +14,145.60451133728927,2.456320980862184,1570.3394113255001,1060.8883168939365,25447,1436.3296139235233,330.62007285504825,0.9731471284007941,0.017528295,0.015629836914595137,-1.564502483,1 +15,146.4683998,2.515722291407223,1597.2121731008717,1112.4936176836861,25696,1425.3875400306174,399.1057512634511,0.960000565,0.016948068180094332,0.015540735,-1.525625489,1 +16,143.05242581805388,2.560655224989249,1599.9863169005826,1172.1436725438837,25579,1549.3733826295704,453.0143680439514,0.9563005269908358,0.016893351103526345,0.015950030429707374,-1.48154135,1 +17,135.1202771462596,2.8307892172783373,1569.4815777128217,1239.5339035040238,27711,1765.6184577050376,501.9120823173725,0.9587443122044614,0.01752492,0.016695284,-1.460044295,1 +18,110.52717024570318,3.1322192672567777,1399.8910411622276,1333.657563608939,25193,2089.4335006789097,570.2385377726345,0.9620380425806904,0.016303310297430415,0.014918823785791438,-1.461791497,1 +19,98.77471085226807,2.8490279714543516,1300.5208350422442,1431.7246739397917,24382,2009.822870125348,591.4533043412637,0.9557188453667624,0.016406061805003237,0.015382421819796903,-1.475126606,1 +20,96.65796209,2.762764065,1243.182535895545,1532.205613152301,23053,1909.2996274999864,570.8418396764675,0.9542594699070676,0.015835541,0.014886579670613518,-1.477452504,1 +21,103.53763681036408,2.7152110788474424,1143.0661603752512,1618.784409202591,22385,1945.8468572883698,507.72595588042026,0.9653582607237355,0.015258522380643047,0.014130276,-1.502077626,1 +22,118.01004431314624,2.8784553703312934,1085.1069001899134,1685.5675458957587,23695,1806.9850632551056,396.7379047234869,0.975599464,0.016557228,0.015689226841257734,-1.528717894,1 +23,129.20605403602403,3.017219813208806,1038.8894262841895,1731.373373915944,23984,1818.2671566084891,314.2611743246872,0.9849506984154395,0.016035579945897303,0.014601926426601028,-1.559959204,1 +24,135.8110538886401,3.002276459173011,1038.6512539184953,1769.9041274817137,26796,1880.003063963754,234.68000107834985,0.9921781971336686,0.019877733689108862,0.016732106,1.565312819748574,1 +25,134.7705100601381,2.835251317840968,1065.0763234093104,1803.4982923750836,26938,1971.6120853552945,206.10568715503157,0.994521037,0.021857187,0.019505082,1.560940594785825,1 +26,135.8118811881188,2.746854760664036,1140.9147249311397,1833.8115089704459,26866,1948.1242950670853,193.77654435784578,0.9950407349370383,0.027376992,0.024386834413521415,1.5611907086684538,1 +27,136.29323643344418,2.6131774707757707,1176.1535771828185,1864.7247266120462,29171,1997.1778180846798,186.5934200377037,0.9956259873847112,0.039663504028760156,0.03825434,1.5630970082554827,1 +28,137.9972824285809,2.5921985815602837,1215.3122555842779,1893.5789421356135,30174,1995.0253687143545,178.86417276177949,0.9959728738255725,0.043271071595023844,0.039789855129797685,1.5662371652121676,2 +29,139.2128216,2.4631123709841547,1249.168156708824,1922.333151620875,27516,1930.5866628171016,180.51982473770968,0.9956187945648642,0.043857957,0.03691119,1.5670533757504224,2 +30,140.14129380263105,2.3870930187434514,1257.8972796771313,1951.120105553184,25769,2057.554034389048,178.3006270352641,0.9962382418430051,0.046873693,0.044098571,1.5706144410711658,2 +31,139.8919714504617,2.3242872696479697,1217.7774507036575,1979.4952215815154,24799,2049.4089490916954,161.90634518383558,0.9968744985032624,0.047424262,0.042475096257930146,-1.568275674,2 +32,141.8303104662328,2.2848074255841246,1227.196095166969,2002.9058465806038,18746,1981.9681389505768,160.1910766683871,0.9967283689978219,0.043040922,0.041209600478353196,-1.569023672,2 +33,120.96683389074694,2.3111761426978816,1224.6010312151616,1996.875557413601,7176,2026.4426557565876,155.89739285809944,0.9970363760767276,0.017226272,0.015495840999196703,-1.569909106,3 +34,117.67680921052632,2.283717105263158,1218.3983004385964,2002.9114583333333,3648,2063.2689767422125,144.08370834376743,0.9975587148731965,0.010858532490765188,0.00993199,-1.569093045,3 +35,111.65413533834587,2.3038847117794488,1202.3001253132832,2009.186717,1596,2093.8368808490773,124.38872930805596,0.9982338399262116,0.005102449,0.004488025,-1.570296429,3 +36,108.42250922509226,2.263837638376384,1241.7343173431734,2016.7564575645756,542,2084.227657055674,105.61011791566496,0.9987153944664529,0.002292705,0.001944555,-1.569710217,3 +37,102.45038167938931,2.5114503816793894,1202.2824427480916,2017.2213740458014,131,2082.377100579906,96.76338950178457,0.99891979,0.000814368,0.000656194,1.5679661674945196,3 +38,97.29850746268657,2.2686567164179103,1284.7611940298507,2012.955223880597,67,2156.725360764424,97.29765365095275,0.9989818630700821,0.000402736,0.000337967,-1.567633091,3 +39,92.75,3.875,1186.75,2025.5,8,2204.4711436093553,69.06502004388332,0.9995091101407876,0.000165347,7.931550717309619e-05,-1.556173619,3 +40,95.85714285714286,3.2857142857142856,1077.4285714285713,2030.7142857142858,7,1922.3139126572953,64.31044587855887,0.9994402346704361,0.000168019,9.315943571998935e-05,1.5654590761731746,3 diff --git a/tools/visinity/vis-postprocess.xml b/tools/visinity/vis-postprocess.xml new file mode 100644 index 0000000..a17765b --- /dev/null +++ b/tools/visinity/vis-postprocess.xml @@ -0,0 +1,67 @@ + + Visualize pre-processed Visinity archives + + macros.xml + + + + + + 8000 + / + + + + + + + + + + + + + + + + + + + Active InteractiveTools + +Once the Visinity dashboard is launched, see the Visinity publication (https://doi.org/10.1109%2FTVCG.2022.3209378) for navigating Visinity and conducting analyses. + ]]> + + + diff --git a/tools/visinity/vis-preprocess.xml b/tools/visinity/vis-preprocess.xml new file mode 100644 index 0000000..2e6e27e --- /dev/null +++ b/tools/visinity/vis-preprocess.xml @@ -0,0 +1,122 @@ + + Create an archive of pre-processed Visinity files + + macros.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 +6. **Cell Type CSV** File mapping cell-type indices to cell-type names. Expected column names are phenotypeID, phenotype + +Example: + +.. image:: phenotype_id_example.png + +**~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~** + +**IMPORTANT NOTE** + +**The order of all input collections and dataset names matters!** +**For instance, the first mask in the segmentation collection should have been derived** +**from the first image in the registered image collection, and should be match the first name** +**in the datasets name file, etc.** + +**~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~** + +**Outputs** + +The output of the archive builder is a single dataset in Galaxy that is used as the primary input for the +visualization tool (2) + +**~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~** + +**IMPORTANT NOTE** + +**Do not delete any of the input collections, as they are also required inputs for the visualization tool** + +**~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~** + +]]> + + +