Rewrite check on ambiguity in sfmTransform

src/aliceVision/sfm/utils/alignment.cpp

@@ -608,18 +608,20 @@ void computeNewCoordinateSystemFromCamerasXAxis(const sfmData::SfMData& sfmData,
     Eigen::Vector3d nullestSpace = solver.eigenvectors().col(minCol).real();
     const Eigen::Vector3d referenceAxis = Eigen::Vector3d::UnitY();    
 
-    const double d = nullestSpace.dot(meanRy);
+    // Compute the rotation which rotates nullestSpace onto unitY
+    out_R = Matrix3d(Quaterniond().setFromTwoVectors(nullestSpace, referenceAxis));
+    const double d = (out_R * meanRy).dot(Eigen::Vector3d::UnitY());
     const bool inverseDirection = (d < 0.0);
     // We have an ambiguity on the Y direction, so if our Y axis is not aligned with the Y axis of the scene
     // we inverse the axis.
     if(inverseDirection)
     {
         nullestSpace = -nullestSpace;
+        out_R = Matrix3d(Quaterniond().setFromTwoVectors(nullestSpace, referenceAxis));
     }
+
 
-
-    // Compute the rotation which rotates nullestSpace onto unitY
-    out_R = Matrix3d(Quaterniond().setFromTwoVectors(nullestSpace, referenceAxis));
+
     out_S = 1.0;
     out_t = -out_R * meanCameraCenter;
 }