Implement socket locks as an IPC locking mechnism

Author: kostorr

CMakeLists.txt

@@ -76,7 +76,6 @@ set(SRCS
   src/Factory/ChannelFactory.cxx
   src/DmaChannelBase.cxx
   src/ChannelPaths.cxx
-  src/Driver.cxx
   src/Dummy/DummyDmaChannel.cxx
   src/Dummy/DummyBar.cxx
   src/ExceptionInternal.cxx
@@ -212,7 +211,7 @@ set(TEST_SRCS
   test/TestChannelPaths.cxx
   test/TestCruDataFormat.cxx
   test/TestEnums.cxx
-  test/TestInterprocessLock.cxx
+  #test/TestInterprocessLock.cxx
   test/TestMemoryMappedFile.cxx
   test/TestParameters.cxx
   test/TestPciAddress.cxx

include/ReadoutCard/Driver.h

@@ -7,7 +7,6 @@
 #include "ReadoutCard/CardType.h"
 #include "ReadoutCard/ChannelFactory.h"
 #include "ReadoutCard/DmaChannelInterface.h"
-#include "ReadoutCard/Driver.h"
 #include "ReadoutCard/Exception.h"
 #include "ReadoutCard/Parameters.h"
-#include "ReadoutCard/RegisterReadWriteInterface.h"
+#include "ReadoutCard/RegisterReadWriteInterface.h"

include/ReadoutCard/ReadoutCard.h

@@ -202,6 +202,11 @@ Parameters::CardIdType getOptionCardId(const po::variables_map& map)
   return Parameters::cardIdFromString(string);
 }
 
+std::string getOptionCardIdString(const po::variables_map& map)
+{
+  return getOption(option::cardId, map);
+}
+
 } // namespace Options
 } // namespace Util
 } // namespace roc

src/CommandLineUtilities/Options.cxx

@@ -32,11 +32,12 @@ int getOptionRegisterValue(const boost::program_options::variables_map& map);
 int getOptionChannel(const boost::program_options::variables_map& map);
 ResetLevel::type getOptionResetLevel(const boost::program_options::variables_map& map);
 Parameters::CardIdType getOptionCardId(const boost::program_options::variables_map& map);
+std::string getOptionCardIdString(const boost::program_options::variables_map& map);
 int getOptionRegisterRange(const boost::program_options::variables_map& map);
 
 } // namespace Options
 } // namespace CommandLineUtilities
 } // namespace roc
 } // namespace AliceO2
 
-#endif // ALICEO2_READOUTCARD_OPTIONS_H
+#endif // ALICEO2_READOUTCARD_OPTIONS_H

src/CommandLineUtilities/Options.h

@@ -2,6 +2,7 @@
 ///
 /// \brief Utility that tests ReadoutCard DMA performance
 /// \author Pascal Boeschoten ([email protected])
+/// \author Kostas Alexopoulos ([email protected])
 
 
 #include <chrono>
@@ -35,6 +36,7 @@
 #include "ReadoutCard/MemoryMappedFile.h"
 #include "ReadoutCard/Parameters.h"
 #include "ReadoutCard/ReadoutCard.h"
+#include "time.h"
 #include "Utilities/Hugetlbfs.h"
 #include "Utilities/SmartPointer.h"
 #include "Utilities/Util.h"
@@ -229,14 +231,16 @@ class ProgramDmaBench: public Program
 
       // Create channel buffer
       {
-        driver::freeUnusedChannelBuffers();
-
         if (mBufferSize < mSuperpageSize) {
           throw ParameterException() << ErrorInfo::Message("Buffer size smaller than superpage size");
         }
 
-        std::string bufferName = (b::format("roc-bench-dma_id=%s_chan=%s_pages") % map["id"].as<std::string>()
-            % mOptions.dmaChannel).str();
+        // Add time to the buffer's filename. This way we guard our buffer from being overwritten by another process
+        // as this is before the DMA Channel initialization and no lock protection is in place.
+        std::string bufferName = (b::format("roc-bench-dma_id=%s_chan=%s_%s_pages") % map["id"].as<std::string>()
+            % mOptions.dmaChannel
+            % time(0)).str();
+
         Utilities::HugepageType hugepageType;
         mMemoryMappedFile = Utilities::tryMapFile(mBufferSize, bufferName, !mOptions.noRemovePagesFile, &hugepageType);
 

src/CommandLineUtilities/ProgramDmaBench.cxx

@@ -18,8 +18,8 @@
 #include "Utilities/SmartPointer.h"
 
 namespace b = boost;
-namespace bip = boost::interprocess;
-namespace bfs = boost::filesystem;
+//namespace bip = boost::interprocess;
+//namespace bfs = boost::filesystem;
 using std::cout;
 using std::endl;
 using namespace std::literals;

src/Crorc/CrorcDmaChannel.cxx

@@ -2,11 +2,12 @@
 /// \brief Implementation of the DmaChannelBase class.
 ///
 /// \author Pascal Boeschoten ([email protected])
+/// \author Kostas Alexopoulos ([email protected])
 
 #include <boost/filesystem.hpp>
 #include "DmaChannelBase.h"
 #include <iostream>
-#include "ChannelPaths.h"
+//#include "ChannelPaths.h"
 #include "Common/System.h"
 #include "Utilities/SmartPointer.h"
 #include "Visitor.h"
@@ -47,6 +48,48 @@ void DmaChannelBase::checkParameters(Parameters& parameters)
   }
 }
 
+void DmaChannelBase::freeUnusedChannelBuffer()
+{
+  namespace bfs = boost::filesystem;
+  InfoLogger::InfoLogger logger;
+ 
+  try {
+    Pda::PdaLock lock{}; // We're messing around with PDA buffers so we need this even though we hold the DMA lock
+  } catch (const LockException& exception) {
+    log("Failed to acquire PDA lock", InfoLogger::InfoLogger::Debug);
+    throw;
+  }
+
+  try {
+    std::string pciPath = "/sys/bus/pci/drivers/uio_pci_dma/";
+    if (boost::filesystem::exists(pciPath)) {
+      for (auto &entry : boost::make_iterator_range(bfs::directory_iterator(pciPath), {})) {
+        auto filename = entry.path().filename().string();
+        if (filename.size() == 12) {
+          // The PCI directory names are 12 characters long
+          auto pciAddress = filename.substr(5); // Remove leading '0000:'
+
+          if (PciAddress::fromString(pciAddress) && (pciAddress == mCardDescriptor.pciAddress.toString())) {
+            // This is a valid PCI address and it's *ours*
+            std::string dmaPath("/sys/bus/pci/drivers/uio_pci_dma/" + filename + "/dma");
+            for (auto &entry : boost::make_iterator_range(bfs::directory_iterator(dmaPath), {})) {
+              auto bufferId = entry.path().filename().string();
+              if (bfs::is_directory(entry)) {
+                std::string mapPath = dmaPath + "/" + bufferId + "/map";
+                std::string freePath = dmaPath + "/free";
+                                logger << "Freeing PDA buffer '" + mapPath + "'" << InfoLogger::InfoLogger::endm;
+                AliceO2::Common::System::executeCommand("echo " + bufferId + " > " + freePath);
+              }
+            }
+          }
+        }
+      }
+    }
+  } catch (const boost::filesystem::filesystem_error &e) {
+    logger << "Failed to free buffers: " << e.what() << InfoLogger::InfoLogger::endm;
+    throw;
+  }
+}
 
 DmaChannelBase::DmaChannelBase(CardDescriptor cardDescriptor, Parameters& parameters,
     const AllowedChannels& allowedChannels)
@@ -62,41 +105,24 @@ DmaChannelBase::DmaChannelBase(CardDescriptor cardDescriptor, Parameters& parame
   // Do some basic Parameters validity checks
   checkParameters(parameters);
 
-  // Create parent directories
-  auto paths = getPaths();
-  for (const auto& p : {paths.fifo(), paths.lock()}) {
-    Common::System::makeParentDirectories(p);
-  }
-
-  // Get lock
+  //try to acquire lock
   log("Acquiring DMA channel lock", InfoLogger::InfoLogger::Debug);
-  try {
-    Utilities::resetSmartPtr(mInterprocessLock, paths.lock(), paths.namedMutex());
+  try{
+    Utilities::resetSmartPtr(mInterprocessLock, "Alice_O2_RoC_DMA_" + cardDescriptor.pciAddress.toString() +"_lock");
   }
-  catch (const NamedMutexLockException& exception) {
-    if (parameters.getForcedUnlockEnabled().get_value_or(false)) {
-      log("Failed to acquire DMA channel mutex. Forced unlock enabled - attempting cleanup and retry",
-          InfoLogger::InfoLogger::Warning);
-      // The user has indicated to the driver that it is safe to force the unlock. We do it by deleting the lock.
-      boost::interprocess::named_mutex::remove(paths.namedMutex().c_str());
-      // Try again...
-      Utilities::resetSmartPtr(mInterprocessLock, paths.lock(), paths.namedMutex());
-    } else {
-      log("Failed to acquire DMA channel lock", InfoLogger::InfoLogger::Debug);
-      throw;
-    }
+  catch (const LockException& exception) {
+    log("Failed to acquire DMA channel lock", InfoLogger::InfoLogger::Debug);
+    throw;
   }
+
   log("Acquired DMA channel lock", InfoLogger::InfoLogger::Debug);
+
+  freeUnusedChannelBuffer();
 }
 
 DmaChannelBase::~DmaChannelBase()
 {
   log("Releasing DMA channel lock", InfoLogger::InfoLogger::Debug);
-
-  std::string channelLockPath = "/dev/shm/AliceO2_RoC_" + std::string(getCardDescriptor().pciAddress.toString()) + "_Channel_" + std::to_string((int) getChannelNumber ()) + ".lock";
-  std::string channelMutexPath = "/dev/shm/sem.AliceO2_RoC_" + std::string(getCardDescriptor().pciAddress.toString()) + "_Channel_" + std::to_string((int) getChannelNumber ()) + "_Mutex";
-  bfs::remove(channelLockPath);
-  bfs::remove(channelMutexPath);
 }
 
 void DmaChannelBase::log(const std::string& message, boost::optional<InfoLogger::InfoLogger::Severity> severity)

src/DmaChannelBase.cxx

@@ -15,6 +15,7 @@
 #include "CardDescriptor.h"
 #include "ChannelPaths.h"
 #include "InterprocessLock.h"
+#include "Pda/PdaLock.h"
 #include "ReadoutCard/DmaChannelInterface.h"
 #include "ReadoutCard/Exception.h"
 #include "ReadoutCard/Parameters.h"
@@ -112,6 +113,9 @@ class DmaChannelBase: public DmaChannelInterface
     /// Check the validity of basic parameters
     void checkParameters(Parameters& parameters);
 
+    /// Free device's PDA Channel Buffer
+    void freeUnusedChannelBuffer();
+
     /// Type of the card
     const CardDescriptor mCardDescriptor;