Merge pull request #103 from IDEA-Research/dev

release: 0.9.0

MANIFEST.in

@@ -1,4 +1,5 @@
 recursive-include deepdataspace/server/static *
 recursive-include deepdataspace/server/templates *
+recursive-include deepdataspace/samples *
 global-exclude *.py[cod]
 global-exclude .DS_Store

build_doc.sh

@@ -0,0 +1 @@
+sphinx-apidoc -o docs/api_reference deepdataspace/

ddsop.py

@@ -2,27 +2,25 @@
 """
 ddsop.py
 
-This script adds operation commands to dds service.
-These command will load the same configurations generated by dds command.
-
 Usage: ddsop.py [OPTIONS] COMMAND [ARGS]...
 
 Options:
   --help  Show this message and exit.
 
 Commands:
-  delete_all  Delete all datasets imported before.
-  delete_one  Delete a dataset.
-  import_all  Trigger a background task of importing all datasets in a...
-  import_one  Trigger a background task of importing one dataset.
-  lpexport    Export the labels of a label project.
-  shell       Enter a Python interpreter shell with all dds configs loaded.
-  useradd     Add a user by username.
-  userban     Ban a user by username.
-  userdel     Delete a user by username.
-  useredit    Edit user attributes for a user by username.
-  userreset   Reset password for a user by username.
-  userunban   Unban a user by username.
+  delete_one   Delete a dataset.
+  import_all   Trigger a background task of importing all datasets in a...
+  import_coco  Generate a coco meta file.
+  import_one   Trigger a background task of importing one dataset.
+  lpexport     Export the labels of a label project.
+  migrate      Run a migrate script.
+  shell        Enter a Python interpreter shell with all dds configs loaded.
+  useradd      Add a user by username.
+  userban      Ban a user by username.
+  userdel      Delete a user by username.
+  useredit     Edit user attributes for a user by username.
+  userreset    Reset password for a user by username.
+  userunban    Unban a user by username.
 """
 
 from deepdataspace.scripts import ddsop

deepdataspace/algos/calculate_fnfp.py

@@ -38,13 +38,37 @@ def calculate_thresholds(all_gt: List[List],
     """
     For given IoU thresh, calculate confidence thresh for precisions from 0.0 to 1.0 .
 
-    :param all_gt: All ground truth objects from a subset
-                  [image_id(str), category_id(int), bbox(List[int])], bbox = [x1, y1, x2, y2]
-    :param all_det: All prediction objects from a subset
-                  [image_id(str), category_id(int), bbox(List[int]), conf(float)], bbox = [x1, y1, x2, y2]
-    :param iou_thresh: IoU threshold
-    :return [
-                {"conf_thresh": xxx, "recall": xxx, "precision": xxx, "precision_thresh": xxx}
+    :param all_gt: All ground truth objects from a subset.
+
+        .. code-block:: python
+
+            [
+                # [image_id, category_id, [x1, y1, x2, y2]]
+                [1, 1, [10, 20, 30, 40]],
+            ]
+
+    :param all_det: All prediction objects from a subset.
+
+        .. code-block:: python
+
+            [
+                # [image_id, category_id, [x1, y1, x2, y2], conf]
+                [1, 1, [10, 20, 33, 43], 0.8],
+            ]
+
+    :param iou_thresh: float.
+
+    :return conf thresholds: list of dict.
+
+        .. code-block:: python
+
+            [
+                {
+                    "conf_thresh": 10,
+                    "recall": 10,
+                    "precision": 10,
+                    "precision_thresh": 10.1
+                }
             ]
     """
 
@@ -159,14 +183,17 @@ def calculate_fnfp(all_gt: List[List],
     """
     For given IoU thresh, check the correctness of all predictions in an image.
 
-    :param all_gt: All ground truth objects from a subset
-                  [category_id(int), bbox(List[int])], bbox = [x1, y1, x2, y2]
-    :param all_det: All prediction objects from a subset
-                  [category_id(int), bbox(List[int]), conf(float)], bbox = [x1, y1, x2, y2]
+    :param all_gt:
+        | All ground truth objects from a subset
+        | [category_id(int), bbox(List[int])], bbox = [x1, y1, x2, y2]
+    :param all_det:
+        | All prediction objects from a subset
+        | [category_id(int), bbox(List[int]), conf(float)], bbox = [x1, y1, x2, y2]
     :param iou_thresh: IoU threshold
-    :return (gt_results, det_results)
-            gt_results, list, -1 means FN, otherwise means matched det id
-            det_results, list, 1 means TP, 0 means FP
+    :return tuple of list of int:
+            | (gt_results, det_results)
+            | gt_results, list, -1 means FN, otherwise means matched det id
+            | det_results, list, 1 means TP, 0 means FP
     """
 
     gt_arr = np.array(all_gt[:], dtype=np.float32)

deepdataspace/constants.py

@@ -239,15 +239,15 @@ class ContentEncoding:
     ALL_ = {Plain, Base64}
 
 
-class TSVFileType:
+class DatasetFileType:
     """
-    | TSV dataset related file types.
-    | TSV dataset format may contain multiple files, each of these types:
+    Dataset related file types.
     """
 
-    Embedding = "Embedding"  #: .embd file, used by :class:`deepdataspace.plugins.tsv.process.RankByFlags`.
-    Prediction = "Pred"  #: .pred file, used by :class:`deepdataspace.plugins.tsv.importer.TSVImporter`.
-    GroundTruth = LabelName.GroundTruth  #: .tsv file, used by :class:`deepdataspace.plugins.tsv.importer.TSVImporter`.
+    GroundTruth = LabelName.GroundTruth
+    Prediction = "Pred"
+    Embedding = "Embedding"
+    Meta = "Meta"
 
 
 class LabelProjectStatus:
@@ -286,12 +286,12 @@ class LabelProjectRoles:
     ReviewKinds_ = {Reviewer, ReviewLeader}  #: Roles that take part in the reviewing process.
 
     Levels_ = {
-        Owner: 0,
-        Manager: 1,
-        LabelLeader: 2,
+        Owner       : 0,
+        Manager     : 1,
+        LabelLeader : 2,
         ReviewLeader: 3,
-        Labeler: 4,
-        Reviewer: 5
+        Labeler     : 4,
+        Reviewer    : 5
     }  #: The level of every role, smaller number means higher level.
 
 

deepdataspace/environs.py

@@ -29,11 +29,12 @@
 
 .. data:: DDS_RUNTIME_DIR
 
-    | The runtime directory of dds.
+    The runtime directory of dds.
 
     :default: ``~/.deepdataspace``
 
 .. data:: DDS_DJANGO_DIR
+
     The runtime directory of django which includes django log files.
 
     :default: ``$DDS_RUNTIME_DIR/django``