Any possibility to implement AXUIElementCopyElementAtPosition?

[whisperecean]

I'd like to capture text on mouse hover and process it further. I think it can be achieved through this API (please correct me if I am wrong).

ref: adrianriobo/goax#22
cc: @adrianriobo

[ljluestc]

package main

import (
	"fmt"
	"log"
	"os"
	"time"

	"github.com/adrianriobo/goax/pkg/goax/axapp/api"
	"github.com/adrianriobo/goax/pkg/os/macos"
	"github.com/adrianriobo/goax/pkg/os/macos/axui"
	"github.com/adrianriobo/goax/pkg/util/dump"
	"github.com/go-vgo/robotgo"
)

// Output file for captured text
const outputFile = "/tmp/hover_text.log"

func main() {
	// Ensure accessibility permissions
	if !macos.IsAccessibilityEnabled() {
		fmt.Println("Please enable accessibility permissions for this app in System Settings > Privacy & Security > Accessibility.")
		os.Exit(1)
	}

	// Open output file
	file, err := os.OpenFile(outputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		log.Fatalf("Failed to open output file: %v", err)
	}
	defer file.Close()

	// Logger for file and console
	logger := log.New(os.Stdout, "", log.LstdFlags)
	fileLogger := log.New(file, "", log.LstdFlags)

	// Main loop: Poll mouse position every 500ms
	for {
		// Get current mouse position
		x, y := robotgo.GetMousePos()
		if x == 0 && y == 0 {
			logger.Println("Failed to get mouse position")
			time.Sleep(500 * time.Millisecond)
			continue
		}

		// Get system-wide accessibility element
		systemElement, err := api.GetSystemElement()
		if err != nil {
			logger.Printf("Failed to get system element: %v", err)
			time.Sleep(500 * time.Millisecond)
			continue
		}

		// Get element under mouse
		element, err := axui.GetElementAtPosition(systemElement, float32(x), float32(y))
		if err != nil {
			logger.Printf("Failed to get element at position (%d, %d): %v", x, y, err)
			time.Sleep(500 * time.Millisecond)
			continue
		}

		// Get accessibility attributes
		attributes, err := axui.GetAllAttributes(element)
		if err != nil {
			logger.Printf("Failed to get attributes: %v", err)
			time.Sleep(500 * time.Millisecond)
			continue
		}

		// Extract relevant text attributes (e.g., AXDescription, AXHelp, AXValue)
		var text string
		if desc, ok := attributes["AXDescription"]; ok && desc != nil {
			text = fmt.Sprintf("%v", desc)
		} else if help, ok := attributes["AXHelp"]; ok && help != nil {
			text = fmt.Sprintf("%v", help)
		} else if value, ok := attributes["AXValue"]; ok && value != nil {
			text = fmt.Sprintf("%v", value)
		}

		// Process text if found
		if text != "" {
			processedText := processText(text)
			logger.Printf("Captured text at (%d, %d): %s", x, y, processedText)
			fileLogger.Printf("Captured text at (%d, %d): %s", x, y, processedText)
		}

		// Delay to avoid overwhelming the system
		time.Sleep(500 * time.Millisecond)
	}
}

// processText: Example processing function (e.g., clean or analyze text)
func processText(text string) string {
	// Example: Convert to uppercase and trim
	return strings.ToUpper(strings.TrimSpace(text))
}

[whisperecean]

This is AI generated and does not work