improved make process with more artifacts and all dependency versions included in logs

Author: thomasWeise

Makefile

@@ -158,7 +158,11 @@ create_distribution: static_analysis test create_documentation
 	source "$$tempDir/bin/activate" &&\
 	echo "Now installing tar.gz." &&\
 	python3 -m pip --no-input --timeout 360 --retries 100 --require-virtualenv install "$(CWD)/dist/moptipy-$(VERSION).tar.gz" && ## nosem \
-	echo "Finished, cleaning up." &&\
+	echo "Installing tar.gz has worked. We now create the list of packages in this environment via pip freeze." &&\
+	pip freeze > "$(CWD)/dist/moptipy-$(VERSION)-requirements_frozen.txt" &&\
+	echo "Now fixing moptipy line in requirements file." &&\
+	sed -i "s/^moptipy.*/moptipy==$(VERSION)/" "$(CWD)/dist/moptipy-$(VERSION)-requirements_frozen.txt" &&\
+	echo "Now we deactivate the environment." &&\
 	deactivate &&\
 	rm -rf "$$tempDir" &&\
 	echo "Now testing the wheel." &&\
@@ -169,9 +173,14 @@ create_distribution: static_analysis test create_documentation
 	source "$$tempDir/bin/activate" &&\
 	echo "Now installing wheel." &&\
 	python3 -m pip --no-input --timeout 360 --retries 100 --require-virtualenv install "$(CWD)/dist/moptipy-$(VERSION)-py3-none-any.whl" && ## nosem \
-	echo "Finished, cleaning up." &&\
+	echo "Now we deactivate the environment." &&\
 	deactivate &&\
+	echo "Finished, cleaning up." &&\
 	rm -rf "$$tempDir" &&\
+	echo "Now also packaging the documentation." &&\
+	cd docs/build &&\
+	tar --dereference --exclude=".nojekyll" -c * | xz -v -9e -c > "$(CWD)/dist/moptipy-$(VERSION)-documentation.tar.xz" &&\
+	cd $(CWD) &&\
 	echo "Successfully finished building source distribution."
 
 # We install the package and see if that works out.

moptipy/utils/sys_info.py

@@ -13,10 +13,29 @@
 import moptipy.version as ver
 from moptipy.api import logging
 from moptipy.utils.logger import InMemoryLogger, Logger, KeyValueLogSection, \
-    CSV_SEPARATOR
+    CSV_SEPARATOR, KEY_VALUE_SEPARATOR, SCOPE_SEPARATOR
 from moptipy.utils.path import Path
 
 
+def __cpu_affinity(proc: Optional[psutil.Process] = None) -> Optional[str]:
+    """
+    Get the CPU affinity.
+
+    :param proc: the process handle
+    :return: the CPU affinity string.
+    """
+    if proc is None:
+        proc = psutil.Process()
+    if proc is None:
+        return None
+    cpua = proc.cpu_affinity()
+    if cpua:
+        cpua = CSV_SEPARATOR.join(map(str, cpua))
+        if len(cpua) > 0:
+            return cpua
+    return None
+
+
 # noinspection PyBroadException
 def __make_sys_info() -> str:
     """
@@ -131,10 +150,9 @@ def __get_mem_size() -> Optional[int]:
                 __v(k, logging.KEY_NODE_NAME, platform.node())
                 proc = psutil.Process()
                 __v(k, logging.KEY_PROCESS_ID, hex(proc.pid))
-                cpua = proc.cpu_affinity()
+                cpua = __cpu_affinity(proc)
                 if cpua:
-                    __v(k, logging.KEY_CPU_AFFINITY,
-                        CSV_SEPARATOR.join(map(str, cpua)))
+                    __v(k, logging.KEY_CPU_AFFINITY, cpua)
                 del proc, cpua
 
                 # see https://stackoverflow.com/questions/166506/.
@@ -151,8 +169,11 @@ def __get_mem_size() -> Optional[int]:
 
             with kv.scope(logging.SCOPE_VERSIONS) as k:
                 __v(k, "moptipy", ver.__version__)
-                for package in ["numpy", "numba", "matplotlib",
-                                "psutil", "scikit-learn"]:
+                for package in ["cycler", "fonttools", "joblib", "kiwisolver",
+                                "llvmlite", "matplotlib", "numba", "numpy",
+                                "packaging", "Pillow", "psutil", "pyparsing",
+                                "python-dateutil", "scikit-learn", "scipy",
+                                "six", "threadpoolctl"]:
                     __v(k, package.replace("-", ""),
                         ilm.version(package).strip())
 
@@ -214,7 +235,23 @@ def __make_mhz_str(tpl: Tuple[int, ...]) -> str:
 
 def refresh_sys_info():
     """Refresh the system information."""
-    __SYS_INFO[0] = __make_sys_info()
+    sys_info_str = __SYS_INFO[0]
+    start = f"\n{logging.SCOPE_SESSION}{SCOPE_SEPARATOR}" \
+            f"{logging.KEY_CPU_AFFINITY}{KEY_VALUE_SEPARATOR}"
+    start_i = sys_info_str.find(start)
+    if start_i < 0:
+        return   # no affinity, don't need to update
+    start_i += len(start)
+    end_i = sys_info_str.find("\n", start_i)
+    if end_i <= start_i:
+        raise ValueError(f"Empty {logging.KEY_CPU_AFFINITY}?")
+    affinity = __cpu_affinity()
+    if affinity is None:
+        raise ValueError(
+            f"first affinity query is {sys_info_str[start_i:end_i]},"
+            f" but second one is None?")
+    sys_info_str = f"{sys_info_str[:start_i]}{affinity}{sys_info_str[end_i:]}"
+    __SYS_INFO[0] = sys_info_str
 
 
 def log_sys_info(logger: Logger) -> None: