update changelog and scripts

CHANGELOG.md

@@ -0,0 +1,33 @@
+# task_spatial_simulators 0.1.0
+
+First release of the spatial simulator benchmark.
+
+Core task documentation and API:
+  - Component types: Process Dataset, Method, Metric
+  - File formats: Single-Cell Dataset, Spatial Dataset, Solution, Score
+
+Dataset processing components for fetching datasets from the SpatialSimBench figshare:
+  - Source: https://figshare.com/articles/dataset/SpatialSimBench_dataset/26054188
+  - Transforms the h5ads into standardised components
+
+Simulation methods under `src/methods/`:
+  - `scdesign2`
+  - `scdesign3_nb`
+  - `scdesign3_poisson`
+  - `sparsim`
+  - `splatter`
+  - `srtsim`
+  - `synsim`
+  - `zinbwave`
+
+Control methods under `src/control_methods/`:
+  - `negative_normal`
+  - `negative_shuffle`
+  - `positive`
+
+Metrics under `src/metrics/`:
+  - `ks_statistic_gene_cell`
+  - `ks_statistic_sc_features/`
+
+Documentation:
+  - Check the `README.md` and `INSTRUCTIONS.md` for how to use and extend the benchmark.

scripts/run_benchmark/render_results_local.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# fail on error
+set -e
+
+# ensure we're in the root of the repo
+REPO_ROOT=$(git rev-parse --show-toplevel)
+cd "$REPO_ROOT"
+
+# set input and output directories
+TASK=task_spatial_simulators
+BASE_DIR="s3://openproblems-data/resources/$TASK/results"
+OUTPUT_DIR="output/report"
+
+# find subdir in bucket with latest date
+DATE=$(aws s3 ls $BASE_DIR --recursive | awk '{print $4}' | grep 'task_info.yaml' | sort -r | head -n 1 | sed 's#.*/run_\(.*\)/[^/]*$#\1#')
+
+INPUT_DIR="$BASE_DIR/run_$DATE"
+TASK_STRIP_PREFIX=$(echo $TASK | sed 's/task_//')
+
+echo "Processing $DATE -> $OUTPUT_DIR"
+
+
+# start the run
+extra_filters=()
+# extra_filters=(
+#   --datasets_exclude "cellxgene_census/hypomap;cellxgene_census/mouse_pancreas_atlas"
+#   --metrics_exclude "hvg_overlap"
+# )
+
+nextflow run openproblems-bio/openproblems \
+  -r build/main \
+  -main-script target/nextflow/reporting/process_task_results/main.nf \
+  -profile docker \
+  -resume \
+  -latest \
+  -c common/nextflow_helpers/labels_ci.config \
+  --id "$TASK/run_$DATE" \
+  --input_scores "$INPUT_DIR/score_uns.yaml" \
+  --input_dataset_info "$INPUT_DIR/dataset_uns.yaml" \
+  --input_method_configs "$INPUT_DIR/method_configs.yaml" \
+  --input_metric_configs "$INPUT_DIR/metric_configs.yaml" \
+  --input_trace "$INPUT_DIR/trace.txt" \
+  --input_task_info "$INPUT_DIR/task_info.yaml" \
+  --output_state '$id/state.yaml' \
+  --output_combined '$id/combined_output.json' \
+  --output_report '$id/report.html' \
+  --output_dataset_info '$id/dataset_info.json' \
+  --output_method_info '$id/method_info.json' \
+  --output_metric_info '$id/metric_info.json' \
+  --output_results '$id/results.json' \
+  --output_quality_control '$id/quality_control.json' \
+  --publish_dir "$OUTPUT_DIR" \
+  "${extra_filters[@]}"

scripts/run_benchmark/run_full_seqeracloud.sh

@@ -21,11 +21,10 @@ publish_dir: "$publish_dir"
 HERE
 
 tw launch https://github.com/openproblems-bio/task_spatial_simulators.git \
-  --revision build/main \
+  --revision v0.1.0 \
   --pull-latest \
   --main-script target/nextflow/workflows/run_benchmark/main.nf \
   --workspace 53907369739130 \
-  --compute-env 5DwwhQoBi0knMSGcwThnlF \
   --params-file /tmp/params.yaml \
   --entry-name auto \
   --config common/nextflow_helpers/labels_tw.config \