src/DeviceManager.cc

@@ -224,7 +224,7 @@ namespace ChimeraTK {
       }
 
       /****************************************************************************************************************/
-      // Sync point (stage OPEN complete): Device opened. Synchronise before stating init scripts.
+      // Sync point (stage OPEN complete): Device opened. Synchronise before starting init scripts.
       /****************************************************************************************************************/
       assert(_recoveryGroup->errorAtStage ==
           RecoveryGroup::RecoveryStage::NO_ERROR); // no other thread must have modified the flag until here.

tests/executables_src/testRecoveryGroups.cc

@@ -15,6 +15,7 @@
 #include <ChimeraTK/NDRegisterAccessor.h>
 #include <ChimeraTK/ScalarRegisterAccessor.h>
 #include <ChimeraTK/VoidRegisterAccessor.h>
+
 namespace ctk = ChimeraTK;
 
 #include <boost/smart_ptr/shared_ptr.hpp>
@@ -32,8 +33,8 @@ struct DeviceModuleWithPath : public ctk::ModuleGroup {
   ctk::DeviceModule dev;
 };
 
-// A test application with 7 devices in 3 recovery groups.
-// It is used in all tests, and extended with initialisation handlers and further backends where needed.
+// A test application with 4 devices in 2 recovery groups.
+// It is used in most tests, and extended with initialisation handlers where needed.
 struct BasicTestApp : ctk::Application {
   explicit BasicTestApp(const std::string& name = "BasicTestApp") : Application(name) {}
   ~BasicTestApp() override { shutdown(); }
@@ -65,7 +66,10 @@ struct Fixture {
 
 /**********************************************************************************************************************/
 
-// Test that App1 actually has two different recovery groups: [Use1, Use2, Use12], [Use3]
+// Spec XXX: Devices which have common backends are recovered together
+// Spec XXY: Devices which don't share backends are recovered independently
+//
+// Note: the tests are done together because test XXY requires exactly the same lines of code as the XXX test
 BOOST_FIXTURE_TEST_CASE(TestRecoveryGroups, Fixture<BasicTestApp>) {
   // Pre-condition: wait until all devices are ok
   // Necessary because we are not using the testable mode
@@ -80,11 +84,12 @@ BOOST_FIXTURE_TEST_CASE(TestRecoveryGroups, Fixture<BasicTestApp>) {
 
   trigger.write();
 
-  // The actual test: Check that Use1, Use2 and Use12 are in error state, while Use3 is still working
+  // The actual test: XXX Check that Use1, Use2 and Use12 are in the same recovery group and thus have seen the error
   for(auto const* dev : {"Use1", "Use2", "Use12"}) {
     CHECK_TIMEOUT(testFacility.readScalar<int>(std::string("Devices/") + dev + "/status") == 1, 10000);
   }
 
+  // Test XXY: Use3 is in a different recovery and still OK
   CHECK_TIMEOUT(testFacility.readScalar<int>("Devices/Use3/status") == 0, 10000);
 
   // Remove error condition on raw1 and recover everything
@@ -97,7 +102,7 @@ BOOST_FIXTURE_TEST_CASE(TestRecoveryGroups, Fixture<BasicTestApp>) {
 }
 
 /**********************************************************************************************************************/
-// Spec ????: all devices have to succesfully complete the OPEN stage before any one starts the init handlers.
+// Spec ????: all devices have to succesfully completed the OPEN stage before any one starts the init handlers.
 BOOST_FIXTURE_TEST_CASE(TestRecoveryStepOpen, Fixture<BasicTestApp>) {
   // pre-condition: all (relevant) devices OK
   for(auto const* dev : {"Use1", "Use2"}) {
@@ -357,3 +362,66 @@ BOOST_FIXTURE_TEST_CASE(TestRecoveryWriteFailure, Fixture<RecoveryFailureTestApp
 }
 
 /**********************************************************************************************************************/
+
+struct IncompleteRecoveryTestApp : ctk::Application {
+  explicit IncompleteRecoveryTestApp() : Application("IncompleteRecoveryTestApp") {}
+  ~IncompleteRecoveryTestApp() override { shutdown(); }
+
+  ctk::SetDMapFilePath path{"recoveryGroups.dmap"};
+
+  std::atomic<bool> throwInInit{false};
+  std::atomic<size_t> initCounter{0};
+  std::barrier<> aboutToThrow{2};
+  void init() {
+    // cheap implementation with busy waiting
+    if(throwInInit) {
+      if(++initCounter == 2) {
+        // The other init handler has passed this point already. Wait a bit to be pretty sure it has reached
+        // the INIT_HANDLER barrier.
+        usleep(100000); // 100 ms
+
+        // Tell the test thread that we are here, about to throw the exception
+        (void)aboutToThrow.arrive();
+
+        // Jump out of the DeviceManager main loop with a thread_interrupted exception, just like all other
+        // breadpoints do
+        throw boost::thread_interrupted(); // NOLINT hicpp-exception-baseclass
+      }
+    }
+  }
+
+  // recovery group with Use1 and Use2
+  DeviceModuleWithPath singleDev1{this, "Use1"};
+  DeviceModuleWithPath singleDev2{this, "Use2"};
+  DeviceModuleWithPath mappedDev12{this, "Use12ReadOnly"};
+};
+
+/**********************************************************************************************************************/
+
+// Test Spec ???: Application exits cleanly if recovery is incomplete, in particular if a DeviceManager is waiting at
+// a barrier, and another DeviceManager in the same recovery group quits the main loop.
+BOOST_AUTO_TEST_CASE(TestIncompleteRecovery) {
+  { // open a new scope so we can test after the app goes out of scope
+    IncompleteRecoveryTestApp testApp;
+    ChimeraTK::TestFacility testFacility{testApp, /* enableTestableMode */ false};
+    testFacility.runApplication();
+
+    // pre-condition: all (relevant) devices OK
+    for(auto const* dev : {"Use1", "Use2"}) {
+      CHECK_TIMEOUT(testFacility.readScalar<int>(std::string("Devices/") + dev + "/status") == 0, 10000);
+    }
+
+    testApp.throwInInit = true;
+    testApp.singleDev1.dev.reportException("reported from test");
+
+    // Wait until the init handler which will throw told us it has reached that point, so we don't end the application
+    // scope before the test is sensitive.
+    testApp.aboutToThrow.arrive_and_wait();
+
+    // now end the scope of the application
+  }
+  // The actual test: We reached this point, the test did not block
+  BOOST_CHECK(true);
+}
+
+/**********************************************************************************************************************/