[visionOS] View Spatial Photos on the Web with zero author adoption

https://bugs.webkit.org/show_bug.cgi?id=277447 <rdar://130545126> Reviewed by Aditya Keerthi. Add a new context menu item to spatial photos on visionOS to view them spatially. * Source/WTF/wtf/PlatformEnableCocoa.h: Introduce new flag for feature. * Source/WebCore/platform/graphics/BitmapImage.h: * Source/WebCore/platform/graphics/BitmapImageDescriptor.cpp: (WebCore::BitmapImageDescriptor::isSpatial const): * Source/WebCore/platform/graphics/BitmapImageDescriptor.h: * Source/WebCore/platform/graphics/BitmapImageSource.h: * Source/WebCore/platform/graphics/Image.h: (WebCore::Image::isSpatial const): * Source/WebCore/platform/graphics/ImageDecoder.h: (WebCore::ImageDecoder::isSpatial const): * Source/WebCore/platform/graphics/ImageSource.h: (WebCore::ImageSource::isSpatial const): * Source/WebKit/Shared/ios/InteractionInformationAtPosition.h: * Source/WebKit/Shared/ios/InteractionInformationAtPosition.serialization.in: * Source/WebKit/UIProcess/API/Cocoa/_WKActivatedElementInfo.h: * Source/WebKit/UIProcess/API/Cocoa/_WKActivatedElementInfo.mm: (-[_WKActivatedElementInfo _initWithInteractionInformationAtPosition:isUsingAlternateURLForImage:userInfo:]): (-[_WKActivatedElementInfo isSpatialImage]): * Source/WebKit/WebProcess/WebPage/ios/WebPageIOS.mm: (WebKit::imagePositionInformation): Plumb spatial photo detection from image load to hit-test to activated element. * Source/WebKit/Shared/ios/GestureTypes.h: * Source/WebKit/UIProcess/API/Cocoa/_WKElementAction.h: * Source/WebKit/UIProcess/API/Cocoa/_WKElementAction.mm: (+[_WKElementAction _elementActionWithType:customTitle:assistant:disabled:]): (+[_WKElementAction imageForElementActionType:]): (elementActionTypeToUIActionIdentifier): (uiActionIdentifierToElementActionType): * Source/WebKit/UIProcess/ios/WKActionSheetAssistant.mm: (-[WKActionSheetAssistant defaultActionsForImageSheet:]): (-[WKActionSheetAssistant handleElementActionWithType:element:needsInteraction:]): * Source/WebKit/WebProcess/WebPage/ios/WebPageIOS.mm: (WebKit::WebPage::performActionOnElement): * Source/WebCore/en.lproj/Localizable.strings: Add context menu item to spatial photos to view the photo spatially. Canonical link: https://commits.webkit.org/282016@main
hoffmanjoshua · Aug 8, 2024 · 6a05886 · 6a05886
1 parent 7eb3b2d
commit 6a05886
Show file tree

Hide file tree

Showing 17 changed files with 106 additions and 1 deletion.
diff --git a/Source/WebCore/en.lproj/Localizable.strings b/Source/WebCore/en.lproj/Localizable.strings
@@ -1684,6 +1684,9 @@
 /* Validation message for input form controls with value higher than allowed maximum */
 "Value must be less than or equal to %s" = "Value must be less than or equal to %s";
 
+/* Title for View Spatial Photo action button */
+"View Spatial Photo" = "View Spatial Photo";
+
 /* Codec Strings */
 "Vorbis Codec String" = "Vorbis";
 

diff --git a/Source/WebCore/platform/graphics/BitmapImage.h b/Source/WebCore/platform/graphics/BitmapImage.h
@@ -118,6 +118,10 @@ class BitmapImage final : public Image {
     bool shouldUseQuickLookForFullscreen() const final { return m_source->shouldUseQuickLookForFullscreen(); }
 #endif
 
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    bool isSpatial() const final { return m_source->isSpatial(); }
+#endif
+
     // Image methods
     bool isBitmapImage() const final { return true; }
     bool isAnimating() const final { return m_source->isAnimating(); }

diff --git a/Source/WebCore/platform/graphics/BitmapImageDescriptor.cpp b/Source/WebCore/platform/graphics/BitmapImageDescriptor.cpp
@@ -248,6 +248,15 @@ bool BitmapImageDescriptor::shouldUseQuickLookForFullscreen() const
 }
 #endif
 
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+bool BitmapImageDescriptor::isSpatial() const
+{
+    if (RefPtr decoder = m_source.decoderIfExists())
+        return decoder->isSpatial();
+    return false;
+}
+#endif
+
 void BitmapImageDescriptor::dump(TextStream& ts) const
 {
     ts.dumpProperty("size", size());

diff --git a/Source/WebCore/platform/graphics/BitmapImageDescriptor.h b/Source/WebCore/platform/graphics/BitmapImageDescriptor.h
@@ -68,6 +68,10 @@ class BitmapImageDescriptor {
     bool shouldUseQuickLookForFullscreen() const;
 #endif
 
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    bool isSpatial() const;
+#endif
+
     void dump(TextStream&) const;
 
 private:

diff --git a/Source/WebCore/platform/graphics/BitmapImageSource.h b/Source/WebCore/platform/graphics/BitmapImageSource.h
@@ -167,6 +167,10 @@ class BitmapImageSource : public ImageSource {
     bool shouldUseQuickLookForFullscreen() const final { return m_descriptor.shouldUseQuickLookForFullscreen(); }
 #endif
 
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    bool isSpatial() const final { return m_descriptor.isSpatial(); }
+#endif
+
     // ImageFrame metadata
     ImageOrientation frameOrientationAtIndex(unsigned index) const final;
 

diff --git a/Source/WebCore/platform/graphics/Image.h b/Source/WebCore/platform/graphics/Image.h
@@ -162,6 +162,10 @@ class Image : public RefCounted<Image>, public CanMakeWeakPtr<Image> {
     virtual bool shouldUseQuickLookForFullscreen() const { return false; }
 #endif
 
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    virtual bool isSpatial() const { return false; }
+#endif
+
     virtual void dump(WTF::TextStream&) const;
 
     WEBCORE_EXPORT RefPtr<ShareableBitmap> toShareableBitmap() const;

diff --git a/Source/WebCore/platform/graphics/ImageDecoder.h b/Source/WebCore/platform/graphics/ImageDecoder.h
@@ -94,6 +94,10 @@ class ImageDecoder : public ThreadSafeRefCounted<ImageDecoder> {
     virtual bool shouldUseQuickLookForFullscreen() const { return false; }
 #endif
 
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    virtual bool isSpatial() const { return false; }
+#endif
+
     virtual IntSize frameSizeAtIndex(size_t, SubsamplingLevel = SubsamplingLevel::Default) const = 0;
     virtual bool frameIsCompleteAtIndex(size_t) const = 0;
     virtual ImageOrientation frameOrientationAtIndex(size_t) const { return ImageOrientation::Orientation::None; }

diff --git a/Source/WebCore/platform/graphics/ImageSource.h b/Source/WebCore/platform/graphics/ImageSource.h
@@ -94,6 +94,10 @@ class ImageSource : public ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr<Image
     virtual bool shouldUseQuickLookForFullscreen() const { return false; }
 #endif
 
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    virtual bool isSpatial() const { return false; }
+#endif
+
     // ImageFrame Metadata
     virtual Seconds frameDurationAtIndex(unsigned) const { RELEASE_ASSERT_NOT_REACHED(); return 0_s; }
     virtual ImageOrientation frameOrientationAtIndex(unsigned) const { RELEASE_ASSERT_NOT_REACHED(); return ImageOrientation::Orientation::None; }

diff --git a/Source/WebKit/Shared/ios/GestureTypes.h b/Source/WebKit/Shared/ios/GestureTypes.h
@@ -60,7 +60,10 @@ enum class SheetAction : uint8_t {
     Copy,
     SaveImage,
     PauseAnimation,
-    PlayAnimation
+    PlayAnimation,
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    ViewSpatial
+#endif
 };
 
 enum class SelectionFlags : uint8_t {

diff --git a/Source/WebKit/Shared/ios/InteractionInformationAtPosition.h b/Source/WebKit/Shared/ios/InteractionInformationAtPosition.h
@@ -89,6 +89,9 @@ struct InteractionInformationAtPosition {
     bool shouldNotUseIBeamInEditableContent { false };
     bool isImageOverlayText { false };
     bool isVerticalWritingMode { false };
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    bool isSpatialImage { false };
+#endif
     WebCore::FloatPoint adjustedPointForNodeRespondingToClickEvents;
     URL url;
     URL imageURL;

diff --git a/Source/WebKit/Shared/ios/InteractionInformationAtPosition.serialization.in b/Source/WebKit/Shared/ios/InteractionInformationAtPosition.serialization.in
@@ -64,6 +64,9 @@ struct WebKit::InteractionInformationAtPosition {
     bool shouldNotUseIBeamInEditableContent;
     bool isImageOverlayText;
     bool isVerticalWritingMode;
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    bool isSpatialImage;
+#endif
     WebCore::FloatPoint adjustedPointForNodeRespondingToClickEvents;
     URL url;
     URL imageURL;

diff --git a/Source/WebKit/UIProcess/API/Cocoa/_WKActivatedElementInfo.h b/Source/WebKit/UIProcess/API/Cocoa/_WKActivatedElementInfo.h
@@ -48,6 +48,9 @@ WK_CLASS_AVAILABLE(macos(10.10), ios(8.0))
 @property (nonatomic, readonly) CGRect boundingRect;
 @property (nonatomic, readonly) NSString *ID WK_API_AVAILABLE(macos(10.12), ios(10.0));
 @property (nonatomic, readonly) BOOL isAnimatedImage WK_API_AVAILABLE(macos(10.15), ios(13.0));
+#if defined(TARGET_OS_VISION) && TARGET_OS_VISION && __VISION_OS_VERSION_MIN_REQUIRED >= 20000
+@property (nonatomic, readonly) BOOL isSpatialImage WK_API_AVAILABLE(visionos(WK_XROS_TBA));
+#endif // defined(TARGET_OS_VISION) && TARGET_OS_VISION & __VISION_OS_VERSION_MIN_REQUIRED >= 20000
 #if TARGET_OS_IPHONE
 @property (nonatomic, readonly) BOOL isAnimating;
 @property (nonatomic, readonly) BOOL canShowAnimationControls;

diff --git a/Source/WebKit/UIProcess/API/Cocoa/_WKActivatedElementInfo.mm b/Source/WebKit/UIProcess/API/Cocoa/_WKActivatedElementInfo.mm
@@ -54,6 +54,9 @@ @implementation _WKActivatedElementInfo  {
     BOOL _animatedImage;
     BOOL _isImage;
     BOOL _isUsingAlternateURLForImage;
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    BOOL _isSpatialImage;
+#endif
 }
 
 #if PLATFORM(IOS_FAMILY)
@@ -90,6 +93,9 @@ - (instancetype)_initWithInteractionInformationAtPosition:(const WebKit::Interac
     _canShowAnimationControls = information.canShowAnimationControls;
     _isImage = information.isImage;
     _isUsingAlternateURLForImage = isUsingAlternateURLForImage;
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    _isSpatialImage = information.isSpatialImage;
+#endif
     _userInfo = userInfo;
 #if ENABLE(ACCESSIBILITY_ANIMATION_CONTROL)
     _animationsUnderElement = information.animationsAtPoint;
@@ -200,6 +206,13 @@ - (BOOL)_isUsingAlternateURLForImage
     return _isUsingAlternateURLForImage;
 }
 
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+- (BOOL)isSpatialImage
+{
+    return _isSpatialImage;
+}
+#endif
+
 #if PLATFORM(IOS_FAMILY)
 - (BOOL)isAnimating
 {

diff --git a/Source/WebKit/UIProcess/API/Cocoa/_WKElementAction.h b/Source/WebKit/UIProcess/API/Cocoa/_WKElementAction.h
@@ -56,6 +56,9 @@ typedef NS_ENUM(NSInteger, _WKElementActionType) {
     _WKElementActionTypeImageExtraction WK_API_AVAILABLE(ios(15.0)),
     _WKElementActionTypeRevealImage WK_API_AVAILABLE(ios(15.0)),
     _WKElementActionTypeCopyCroppedImage WK_API_AVAILABLE(ios(16.0)),
+#if defined(TARGET_OS_VISION) && TARGET_OS_VISION && __VISION_OS_VERSION_MIN_REQUIRED >= 20000
+    _WKElementActionTypeViewSpatial WK_API_AVAILABLE(visionos(WK_XROS_TBA)),
+#endif
     _WKElementActionPlayAnimation,
     _WKElementActionPauseAnimation,
 } WK_API_AVAILABLE(macos(10.10), ios(8.0));

diff --git a/Source/WebKit/UIProcess/API/Cocoa/_WKElementAction.mm b/Source/WebKit/UIProcess/API/Cocoa/_WKElementAction.mm
@@ -51,6 +51,9 @@
 static UIActionIdentifier const WKElementActionTypeOpenIdentifier = @"WKElementActionTypeOpen";
 static UIActionIdentifier const WKElementActionTypeCopyIdentifier = @"WKElementActionTypeCopy";
 static UIActionIdentifier const WKElementActionTypeSaveImageIdentifier = @"WKElementActionTypeSaveImage";
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+static UIActionIdentifier const WKElementActionTypeViewSpatialIdentifier = @"WKElementActionTypeViewSpatial";
+#endif
 #if TARGET_OS_IOS || (defined(TARGET_OS_VISION) && TARGET_OS_VISION)
 static UIActionIdentifier const WKElementActionTypeAddToReadingListIdentifier = @"WKElementActionTypeAddToReadingList";
 static UIActionIdentifier const WKElementActionTypeOpenInDefaultBrowserIdentifier = @"WKElementActionTypeOpenInDefaultBrowser";
@@ -160,6 +163,14 @@ + (instancetype)_elementActionWithType:(_WKElementActionType)type customTitle:(N
             [assistant handleElementActionWithType:type element:actionInfo needsInteraction:YES];
         };
         break;
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    case _WKElementActionTypeViewSpatial:
+        title = WEB_UI_STRING("View Spatial Photo", "Title for View Spatial Photo action button");
+        handler = ^(WKActionSheetAssistant *assistant, _WKActivatedElementInfo *actionInfo) {
+            [assistant handleElementActionWithType:type element:actionInfo needsInteraction:YES];
+        };
+        break;
+#endif
 #if HAVE(SAFARI_SERVICES_FRAMEWORK)
     case _WKElementActionTypeAddToReadingList:
         title = WEB_UI_STRING("Add to Reading List", "Title for Add to Reading List action button");
@@ -275,6 +286,10 @@ + (UIImage *)imageForElementActionType:(_WKElementActionType)actionType
         return [UIImage systemImageNamed:@"doc.on.doc"];
     case _WKElementActionTypeSaveImage:
         return [UIImage systemImageNamed:@"square.and.arrow.down"];
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    case _WKElementActionTypeViewSpatial:
+        return [UIImage systemImageNamed:@"cube"];
+#endif
 #if HAVE(LINK_PREVIEW)
     case _WKElementActionTypeAddToReadingList:
         return [UIImage systemImageNamed:@"eyeglasses"];
@@ -325,6 +340,10 @@ UIActionIdentifier elementActionTypeToUIActionIdentifier(_WKElementActionType ac
         return WKElementActionTypeCopyIdentifier;
     case _WKElementActionTypeSaveImage:
         return WKElementActionTypeSaveImageIdentifier;
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    case _WKElementActionTypeViewSpatial:
+        return WKElementActionTypeViewSpatialIdentifier;
+#endif
 #if HAVE(LINK_PREVIEW)
     case _WKElementActionTypeAddToReadingList:
         return WKElementActionTypeAddToReadingListIdentifier;
@@ -366,6 +385,10 @@ static _WKElementActionType uiActionIdentifierToElementActionType(UIActionIdenti
         return _WKElementActionTypeCopy;
     if ([identifier isEqualToString:WKElementActionTypeSaveImageIdentifier])
         return _WKElementActionTypeSaveImage;
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    if ([identifier isEqualToString:WKElementActionTypeViewSpatialIdentifier])
+        return _WKElementActionTypeViewSpatial;
+#endif
 #if HAVE(LINK_PREVIEW)
     if ([identifier isEqualToString:WKElementActionTypeAddToReadingListIdentifier])
         return _WKElementActionTypeAddToReadingList;

diff --git a/Source/WebKit/UIProcess/ios/WKActionSheetAssistant.mm b/Source/WebKit/UIProcess/ios/WKActionSheetAssistant.mm
@@ -610,6 +610,12 @@ - (void)_appendAnimationAction:(NSMutableArray *)actions elementInfo:(_WKActivat
         [defaultActions addObject:[_WKElementAction _elementActionWithType:_WKElementActionTypeSaveImage info:elementInfo assistant:self]];
 
     [defaultActions addObject:[_WKElementAction _elementActionWithType:_WKElementActionTypeCopy info:elementInfo assistant:self]];
+
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    if ([elementInfo isSpatialImage])
+        [defaultActions addObject:[_WKElementAction _elementActionWithType:_WKElementActionTypeViewSpatial info:elementInfo assistant:self]];
+#endif
+
 #if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
     BOOL enableCopySubjectItem = [_delegate respondsToSelector:@selector(actionSheetAssistantShouldIncludeCopySubjectAction:)] && [_delegate actionSheetAssistantShouldIncludeCopySubjectAction:self];
     [defaultActions addObject:[_WKElementAction _elementActionWithType:_WKElementActionTypeCopyCroppedImage info:elementInfo assistant:self disabled:!enableCopySubjectItem]];
@@ -1048,6 +1054,11 @@ - (void)handleElementActionWithType:(_WKElementActionType)type element:(_WKActiv
     case _WKElementActionTypeSaveImage:
         [delegate actionSheetAssistant:self performAction:WebKit::SheetAction::SaveImage];
         break;
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    case _WKElementActionTypeViewSpatial:
+        [delegate actionSheetAssistant:self performAction:WebKit::SheetAction::ViewSpatial];
+        break;
+#endif
     case _WKElementActionTypeShare:
         if (URL(element.imageURL).protocolIsData() && element.image && [delegate respondsToSelector:@selector(actionSheetAssistant:shareElementWithImage:rect:)])
             [delegate actionSheetAssistant:self shareElementWithImage:element.image rect:element.boundingRect];

diff --git a/Source/WebKit/WebProcess/WebPage/ios/WebPageIOS.mm b/Source/WebKit/WebProcess/WebPage/ios/WebPageIOS.mm
@@ -3148,6 +3148,9 @@ static void imagePositionInformation(WebPage& page, Element& element, const Inte
     info.isAnimating = image.isAnimating();
     RefPtr htmlElement = dynamicDowncast<HTMLElement>(element);
     info.elementContainsImageOverlay = htmlElement && ImageOverlay::hasOverlay(*htmlElement);
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    info.isSpatialImage = image.isSpatial();
+#endif
 
     if (request.includeSnapshot || request.includeImageData)
         info.image = createShareableBitmap(renderImage, { screenSize() * page.corePage()->deviceScaleFactor(), AllowAnimatedImages::Yes, UseSnapshotForTransparentImages::Yes });
@@ -3612,6 +3615,10 @@ static void handleAnimationActions(Element& element, uint32_t action)
             return;
         send(Messages::WebPageProxy::SaveImageToLibrary(WTFMove(*handle), authorizationToken));
     }
+#if ENABLE(SPATIAL_IMAGE_DETECTION)
+    else if (static_cast<SheetAction>(action) == SheetAction::ViewSpatial)
+        element->webkitRequestFullscreen();
+#endif
 
     handleAnimationActions(*element, action);
 }