Ficus Release

.github/workflows/deploy.yml

@@ -10,6 +10,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
@@ -23,27 +26,29 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           context: .
-          file: ./Ficus/docker/run/Run.FicusBackend.Dockerfile
-          platforms: linux/amd64,linux/arm64
+          file: ./Ficus/docker/run/Run.RustFicusBackend.Dockerfile
+          platforms: linux/amd64,linux/arm64/v8
           push: true
-          tags: aerooneqq/ficus:latest
+          tags: aerooneqq/ficus:1.0.6,aerooneqq/ficus:latest
 
   publish-ficus-python-package:
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v4
-        
+
       - name: Setup python
         uses: actions/setup-python@v5
         with:
           python-version: '3.10'
-          
+
       - name: Installing deps
-        run: python -m pip install --upgrade twine
-
+        run: |
+          python -m pip install --upgrade twine
+          python -m pip install --upgrade build
+
       - name: Building Ficus
         run: python -m build ${{ github.workspace }}/Ficus/src/python/
-      
+
       - name: Publishing Ficus
         run: python -m twine upload --repository pypi --username __token__ --password ${{ secrets.PYPI_TOKEN }} ${{ github.workspace }}/Ficus/src/python/dist/*

All.sln

@@ -111,6 +111,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "misc", "misc", "{08516726-B
 		Directory.Packages.props = Directory.Packages.props
 		Readme.md = Readme.md
 		.github\workflows\tests.yml = .github\workflows\tests.yml
+		.github\workflows\deploy.yml = .github\workflows\deploy.yml
 	EndProjectSection
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProcfilerLoggerProvider", "Procfiler\src\dotnet\ProcfilerLoggerProvider\ProcfilerLoggerProvider.csproj", "{F50FE66B-3147-4BE6-97EB-4A6889419E34}"
@@ -121,6 +122,16 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FicusKafkaConstants", "Ficu
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProcfilerOnline.Aspire", "Procfiler\src\dotnet\ProcfilerOnline.Aspire\ProcfilerOnline.Aspire.csproj", "{1BBDEB95-1084-4981-AE5A-EFFC74E095E4}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{92C02EAF-93FB-4A2C-A235-8541D893163C}"
+	ProjectSection(SolutionItems) = preProject
+		scripts\macos\generate_grpc_models.sh = scripts\macos\generate_grpc_models.sh
+	EndProjectSection
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "misc", "misc", "{2918C7E3-3FC8-48A5-ADDF-8358E888B40C}"
+	ProjectSection(SolutionItems) = preProject
+		Ficus\Readme.md = Ficus\Readme.md
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -265,5 +276,6 @@ Global
 		{F50FE66B-3147-4BE6-97EB-4A6889419E34} = {D50BD31E-296B-468E-817A-60AF1CE7A759}
 		{B1223284-182E-4C3A-92CD-F59FBD42FFD6} = {0EC07BD3-AB92-48A0-B68B-05B3F2A767A3}
 		{1BBDEB95-1084-4981-AE5A-EFFC74E095E4} = {D50BD31E-296B-468E-817A-60AF1CE7A759}
+		{2918C7E3-3FC8-48A5-ADDF-8358E888B40C} = {5F2E0AEA-6AAF-4130-B486-A5F4F5A8A4BD}
 	EndGlobalSection
 EndGlobal

Ficus/Readme.md

@@ -1,44 +1,46 @@
 ## Ficus
 
-Ficus is a python package where some Process Mining algorithms are implemented.
-Now this project is an active development stage.
+Ficus is a modern process mining toolkit that consists two parts: backend written in Rust and a Python client.
+For Ficus to work you will need to install additional tools, such as `graphviz`.
 
-Algorithms executions in Ficus use pipelines (similar to sklearn):
+Now the main scenario is to create a `Pipeline` that declaratively describes process mining processing pipeline.
+The input of the pipeline is initial context, where usually names event log or a path to `xes` or `bxes` event log is specified.
 
-```python
+For the pipeline to run, Ficus backend should be launched:
+- Either use `__call__` or `execute_docker` methods to pull the Ficus docker image, launch a container with Ficus backend and remove it
+  after pipeline finished its execution
 
-    Pipeline(
-        ReadLogFromXes(),
-        RemoveEventsFromLogPredicate(procfiler_filter_predicate),
-        RemoveEventsFromLogPredicate(lambda x: 'BYTE[]' in x[concept_name]),
-        IfPipeline(should_filter, Pipeline(
-            PosEntropyDirectFilter(threshold, max_events_to_remove=5),
-        )),
-        FilterTracesByCount(min_event_in_trace_count=3),
-        DiscoverActivitiesFromTandemArrays(array_kind=TandemArrayKind.PrimitiveArray,
-                                           activity_in_trace_filter=activity_in_trace_filter,
-                                           activity_level=0),
-        DrawFullActivitiesDiagram(plot_legend=True),
-        CreateLogFromActivities(use_hashes_as_names=use_hashes_as_names),
-        ClearActivities(),
-        DiscoverActivitiesForSeveralLevels([default_class_extractor, strongest_class_extractor_evt],
-                                           discovering_strategy=ActivitiesDiscoveryStrategy.DiscoverFromAllTraces),
-        CalculatePercentageOfUnattachedEvents(),
-        DrawFullActivitiesDiagram(plot_legend=False),
-        CreateLogFromActivities(undef_evt_strategy, use_hashes_as_names=use_hashes_as_names),
-        ClearActivities(),
-        DiscoverActivitiesForSeveralLevels([default_class_extractor],
-                                           discovering_strategy=ActivitiesDiscoveryStrategy.DiscoverFromAllTraces),
-        DrawFullActivitiesDiagram(plot_legend=True),
-        CreateLogsForActivities(class_extractor=default_class_extractor, activity_level=0),
-        ExecuteWithEachActivityLog(Pipeline(
-            SubstituteUnderlyingEvents(),
-            RemoveEventsFromLogPredicate(methods_filter_predicate),
-            FilterTracesByVariants(),
-            TracesDiversityDiagram(plot_legend=True),
-            RemoveLifecycleTransitionsAttributes(),
-            SaveEventLog(activity_log_save_path_creator)
-        )),
-    )(path)
+  ```python
+  Pipeline(
+      UseNamesEventLog(),
+      AddStartEndArtificialEvents(),
+      DiscoverPetriNetHeuristic(dependency_threshold=0.5, loop_length_two_threshold=0.5),
+      EnsureInitialMarking(),
+      AnnotatePetriNetWithTraceFrequency(show_places_names=False, rankdir='TB'),
+  )({
+      'names_event_log': NamesLogContextValue([
+          ["A", "B", "D"],
+          ["A", "B", "C", "B", "D"],
+          ["A", "B", "C", "B", "C", "B", "D"],
+      ])
+  })
+
+  ```
 
-```
+- Or use `execute` method that accepts ficus backend url (that is launched manually) and the initial context.
+
+  ```python
+  Pipeline(
+      UseNamesEventLog(),
+      AddStartEndArtificialEvents(),
+      DiscoverPetriNetHeuristic(dependency_threshold=0.5, loop_length_two_threshold=0.5),
+      EnsureInitialMarking(),
+      AnnotatePetriNetWithTraceFrequency(show_places_names=False, rankdir='TB'),
+  ).execute('localhost:8080', {
+      'names_event_log': NamesLogContextValue([
+          ["A", "B", "D"],
+          ["A", "B", "C", "B", "D"],
+          ["A", "B", "C", "B", "C", "B", "D"],
+      ])
+  })
+  ```

Ficus/docker/run/Run.RustFicusBackend.Dockerfile

@@ -14,7 +14,7 @@ COPY ./bxes/ ./pmide/bxes/
 
 RUN cargo build --manifest-path /pmide/ficus/src/rust/Cargo.toml --release
 
-FROM --platform=linux/amd64 gcr.io/distroless/cc as run
+FROM gcr.io/distroless/cc as run
 EXPOSE 8080
 
 WORKDIR /app

Ficus/src/python/ficus/grpc_pipelines/entry_points/default_pipeline.py

@@ -1,3 +1,10 @@
+import dataclasses
+
+import docker
+from docker import DockerClient
+from docker.errors import DockerException
+from docker.models.containers import Container
+
 from .util import *
 from ...grpc_pipelines.constants import *
 from ...grpc_pipelines.context_values import *
@@ -10,13 +17,48 @@
 from ...legacy.analysis.event_log_analysis_canvas import draw_colors_event_log_canvas
 from ...legacy.analysis.patterns.patterns_models import UndefinedActivityHandlingStrategy
 from ...legacy.pipelines.analysis.patterns.models import AdjustingMode
+from importlib.metadata import version
+
+@dataclasses.dataclass
+class ContainerCreationResult:
+  id: str
+  port: int
 
 
 class Pipeline:
   def __init__(self, *parts):
     self.parts: list['PipelinePart'] = list(parts)
 
-  def execute(self, ficus_backend: str, initial_context: dict[str, ContextValue]) -> GrpcPipelinePartExecutionResult:
+  def __call__(self, initial_context: dict[str, ContextValue]) -> Optional[GrpcPipelinePartExecutionResult]:
+    return self.execute_docker(initial_context)
+
+  def execute_docker(self, initial_context: dict[str, ContextValue]) -> Optional[GrpcPipelinePartExecutionResult]:
+    try:
+      client = docker.from_env()
+    except DockerException as err:
+      print(f"Failed to create docker client, please ensure that docker is installed and up and running, {err}")
+      return None
+
+    res: Optional[ContainerCreationResult] = None
+    try:
+      try:
+        res = _create_and_run_container(client)
+        backend_url = f'localhost:{res.port}'
+
+        print(f'Started Ficus backend container ({backend_url}), executing pipeline')
+
+        self.execute(backend_url, initial_context)
+      except Exception as err:
+        print("Failed to start a ficus backend container", err)
+        return None
+    finally:
+      if res is not None:
+        try:
+          _terminate_container(res.id, client)
+        except Exception as err:
+          print(f'Failed to terminate container {err}')
+
+  def execute(self, ficus_backend: str, initial_context: dict[str, ContextValue]) -> Optional[GrpcPipelinePartExecutionResult]:
     with create_ficus_grpc_channel(ficus_backend) as channel:
       def action(ids):
         stub = GrpcBackendServiceStub(channel)
@@ -52,6 +94,57 @@ def _find_pipeline_parts_with_callbacks(parts) -> list["PipelinePartWithCallback
 
     return result
 
+def _create_and_run_container(client: DockerClient) -> Optional[ContainerCreationResult]:
+  image_name = 'aerooneqq/ficus'
+  image_version = version('ficus_pm')
+  full_image_name = f'{image_name}:{image_version}'
+
+  print(f'Start pulling image {full_image_name}')
+  client.images.pull(image_name, image_version)
+
+  print(f'Pulled Ficus backend image {full_image_name}, starting Ficus backend container')
+
+  container_internal_port = '8080'
+  container = client.containers.run(full_image_name,
+                                    detach=True,
+                                    ports={container_internal_port: 0})
+
+  print(f"Created container for ficus backend: {container.id}, {container.name}")
+
+  container_port_key = f'{container_internal_port}/tcp'
+  print('Waiting until container ports information will be available')
+
+  while not _contains_port(container, container_port_key):
+    container.reload()
+
+  return ContainerCreationResult(container.id, container.ports[container_port_key][0]['HostPort'])
+
+def _contains_port(container: Container, container_port_key: str):
+  return not (len(container.ports) == 0 or
+              container_port_key not in container.ports or
+              len(container.ports[container_port_key]) == 0)
+
+def _terminate_container(container_id: str, client: DockerClient):
+  try:
+    container = client.containers.get(container_id)
+    if container is None:
+      print(f'The container with {container_id} does not exist, can not terminate or remove it')
+      return
+
+    try:
+      container.stop()
+      print(f'Terminated container for ficus backend: {container_id}')
+    except Exception as err:
+      print(f'Failed to stop container {container_id}, {err}')
+
+    try:
+      container.remove(force=True, v=True)
+      print(f'Removed container {container_id}')
+    except Exception as err:
+      print(f'Failed to remove container {container_id}, {err}')
+  except KeyboardInterrupt:
+    _terminate_container(container_id, client)
+
 
 class PipelinePart:
   def __init__(self):
@@ -224,6 +317,7 @@ def append_context_value(config: GrpcPipelinePartConfiguration, key: str, value:
 def append_uint32_value(config: GrpcPipelinePartConfiguration, key: str, value: int):
   append_context_value(config, key, Uint32ContextValue(value))
 
+
 def append_bool_value(config: GrpcPipelinePartConfiguration, key: str, value: bool):
   append_context_value(config, key, BoolContextValue(value))
 
@@ -274,6 +368,7 @@ def append_activity_filter_kind(config: GrpcPipelinePartConfiguration, key: str,
 def append_activities_logs_source(config: GrpcPipelinePartConfiguration, key: str, source: ActivitiesLogsSource):
   append_enum_value(config, key, const_activities_logs_source_enum_name, source.name)
 
+
 def append_json_value(config: GrpcPipelinePartConfiguration, key: str, json_string: str):
   config.configurationParameters.append(GrpcContextKeyValue(
     key=GrpcContextKey(name=key),

Ficus/src/python/requirements.txt

@@ -15,4 +15,5 @@ grpcio-tools==1.59.0
 suffix-tree==0.1.2
 scikit-learn~=1.3.2
 attrs~=23.1.0
-protobuf~=4.25.3
+protobuf~=4.25.3
+docker~=7.1.0