rules.ninja

# This defines the rules on how to build parts of ConceptNet, using the
# Ninja build system:
#
#   http://martine.github.io/ninja/manual.html
#
# Ninja is available in the 'ninja-build' Ubuntu package. It's like make with
# better parallelism and the ability for build steps to produce multiple
# outputs. The tradeoff is that its rule syntax isn't full of magic for
# expanding wildcards and finding dependencies, so in general you have to
# write the dependencies using a script.
#
# This file will become the header of the larger build.ninja file, which also
# contains the programatically-defined dependency graph.

pool memory_hogs
  depth = 1

rule rebuild_ninja
  command = python -m ninja

rule download
  command = python setup.py nltk_download && mkdir -p $prefix && curl -o $out $url

rule extract_tar
  command = tar -xvf $in

rule extract_wiktionary
  command = python -m conceptnet5.readers.extract_wiktionary $in $dir -l $lang && cd conceptnet5/wiktparse && eval "make ${lang}_parser.py"

rule parse
  command = python -m conceptnet5.readers.$parser $in $out

rule parse_globalmind
  command = python -m conceptnet5.readers.globalmind $$(dirname $$(echo $in | cut -d ' ' -f1)) $out

rule parse_dbpedia
  command =python -m conceptnet5.readers.dbpedia $in $out

rule parse_sw
  command = mkdir -p $prefix && eval "python -m conceptnet5.readers.$parser $dir $out ${prefix}${parser}.nt"

rule parse_wiktionary
  command =  python -m conceptnet5.readers.wiktionary -l $lang $in $out

rule msgpack_to_csv
  command = python -m conceptnet5.builders.msgpack_to_csv $in $out

rule collate
  command = cat $in | python -m conceptnet5.builders.distribute_edges -o $dir -n $count

rule count_and_rank
  command = LC_ALL=C sort $in | uniq > $out

rule combine_assertions
  command = python -m conceptnet5.builders.combine_assertions $in $out -l /l/CC/By-SA

rule msgpack_to_assoc
  command = python -m conceptnet5.builders.msgpack_to_assoc $in $out

rule build_db
  command = if [ ! -e ~/.conceptnet5 ]; then ln -s $$(readlink -f $prefix) ~/.conceptnet5; fi && python -m conceptnet5.builders.index_assertions $$(dirname $$(echo $in | cut -d ' ' -f1)) $basename --input-shards $num

rule relation_stats
  command = cut -f 2 $in | LC_ALL=C sort | LC_ALL=C uniq -c | LC_ALL=C sort -nbr > $out

rule dataset_stats
  command = cut -f $fields $in > $out

rule dataset_vs_language_stats
  command = cat $in | sed -r 's:((/[^/\t]+){2})[^\t]*:\1:g' | LC_ALL=C sort | LC_ALL=C uniq -c > $out

rule more_stats
  command = cut -f 2,3,4,9 $in | sed -r 's:((/[^/\t]+){2})[^\t]*:\1:g' | LC_ALL=C sort | LC_ALL=C uniq -c | LC_ALL=C sort -nbr > $out

rule reduce_assoc
  command = python -m conceptnet5.builders.reduce_assoc $$(dirname $$(echo $in | cut -d ' ' -f1))

rule build_assoc
  command = python -m conceptnet5.builders.assoc_to_vector_space $in $$(dirname $$(echo $out | cut -d ' ' -f1))
  pool = memory_hogs

rule msgpack_to_json
  command = python -m conceptnet5.builders.msgpack_to_json $in $out

rule compress_tar
  command = tar jcvf $out $in

rule upload
  command = rsync -Pavz $$(dirname $$(echo $in | cut -d ' ' -f1)) conceptnet5.media.mit.edu:/data/downloads