ODFstudio/odfstudio
2
0
Fork 0
Code Issues Packages Activity

lok: a11y: when we are inside a table notify table and current cell info

Browse Source 
When we get in one or more tables we notify row and column count.
When we get out one or more table we notify we are leaving a table.
When the fosused cell changes we notify the new row/col index.
The paragraph content is notified together with table info, so that
client has some opportunity for getting the screen reader to report
together content and table info.

Change-Id: Ic524259aa1879a70fc3de2467bdee27475352b7d
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/155577
Tested-by: Jenkins
Reviewed-by: Miklos Vajna <vmiklos@collabora.com>
This commit is contained in:
Marco Cecchetti
2023-08-14 08:09:53 +02:00
committed by Miklos Vajna
parent 926c5246b6
commit 59c2e114f7
4 changed files with 304 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
desktop/source/lib/init.cxx
+2
View File
@@ -1798,6 +1798,7 @@ void CallbackFlushHandler::queue(const int type, CallbackData& aCallbackData)
case LOK_CALLBACK_A11Y_FOCUS_CHANGED:
case LOK_CALLBACK_A11Y_CARET_CHANGED:
case LOK_CALLBACK_A11Y_TEXT_SELECTION_CHANGED:
case LOK_CALLBACK_A11Y_FOCUSED_CELL_CHANGED:
case LOK_CALLBACK_COLOR_PALETTES:
{
const auto& pos = std::find(m_queue1.rbegin(), m_queue1.rend(), type);
@@ -1860,6 +1861,7 @@ void CallbackFlushHandler::queue(const int type, CallbackData& aCallbackData)
case LOK_CALLBACK_A11Y_FOCUS_CHANGED:
case LOK_CALLBACK_A11Y_CARET_CHANGED:
case LOK_CALLBACK_A11Y_TEXT_SELECTION_CHANGED:
case LOK_CALLBACK_A11Y_FOCUSED_CELL_CHANGED:
case LOK_CALLBACK_COLOR_PALETTES:
{
if (removeAll(type))
include/LibreOfficeKit/LibreOfficeKitEnums.h
+40 -4
View File
@@ -924,14 +924,17 @@ typedef enum
LOK_CALLBACK_APPLICATION_BACKGROUND_COLOR = 61,
/**
* Accessibility event: a paragraph get focus.
* Accessibility event: a paragraph got focus.
* The payload is a json with the following structure.
*
* {
* "content": "<paragraph text>"
* "position": N
* "start": N1
* "end": N2
* }
* where N is the position of the text cursor inside the focused paragraph.
* where N is the position of the text cursor inside the focused paragraph,
* and [N1,N2] is the range of the text selection inside the focused paragraph.
*/
LOK_CALLBACK_A11Y_FOCUS_CHANGED = 62,
@@ -946,7 +949,7 @@ typedef enum
LOK_CALLBACK_A11Y_CARET_CHANGED = 63,
/**
* Accessibility event: text cursor position has changed.
* Accessibility event: text selection has changed.
*
* {
* "start": N1
@@ -965,7 +968,38 @@ typedef enum
* Informs that the document password has been successfully changed.
* The payload contains the new password and the type.
*/
LOK_CALLBACK_DOCUMENT_PASSWORD_RESET = 66
LOK_CALLBACK_DOCUMENT_PASSWORD_RESET = 66,
/**
* Accessibility event: a cell got focus.
* The payload is a json with the following structure.
*
* {
* "outCount": <number of tables user gets out of>
* "inList": [
* {
* "rowCount": <number of rows for outer table user got in>
* "colCount": <number of columns for outer table user got in>
* },
* ...
* {
* "rowCount": <number of rows for inner table user got in>
* "colCount": <number of columns for inner table user got in>
* }
* ]
* "row": <current row index>
* "col": <current column index>
* "rowSpan": <row span for current cell>
* "colSpan": <column span for current cell>
* "paragraph": {
* <same structure as for LOK_CALLBACK_A11Y_FOCUS_CHANGED>
* }
* }
* where row/column indexes start from 0, inList is the list of tables
* the user got in from the outer to the inner; row/column span default
* value is 1; paragraph is the cell text content.
*/
LOK_CALLBACK_A11Y_FOCUSED_CELL_CHANGED = 67
}
LibreOfficeKitCallbackType;
@@ -1128,6 +1162,8 @@ static inline const char* lokCallbackTypeToString(int nType)
return "LOK_CALLBACK_COLOR_PALETTES";
case LOK_CALLBACK_DOCUMENT_PASSWORD_RESET:
return "LOK_CALLBACK_DOCUMENT_PASSWORD_RESET";
case LOK_CALLBACK_A11Y_FOCUSED_CELL_CHANGED:
return "LOK_CALLBACK_A11Y_FOCUSED_CELL_CHANGED";
}
assert(!"Unknown LibreOfficeKitCallbackType type.");
libreofficekit/source/gtk/lokdocview.cxx
+1
View File
@@ -1491,6 +1491,7 @@ callback (gpointer pData)
case LOK_CALLBACK_A11Y_FOCUS_CHANGED:
case LOK_CALLBACK_A11Y_CARET_CHANGED:
case LOK_CALLBACK_A11Y_TEXT_SELECTION_CHANGED:
case LOK_CALLBACK_A11Y_FOCUSED_CELL_CHANGED:
case LOK_CALLBACK_COLOR_PALETTES:
case LOK_CALLBACK_DOCUMENT_PASSWORD_RESET:
{
sfx2/source/view/viewsh.cxx
+261 -23
View File
@@ -56,6 +56,7 @@
#include <com/sun/star/accessibility/AccessibleStateType.hpp>
#include <com/sun/star/accessibility/AccessibleRole.hpp>
#include <com/sun/star/accessibility/XAccessibleText.hpp>
#include <com/sun/star/accessibility/XAccessibleTable.hpp>
#include <cppuhelper/implbase.hxx>
#include <com/sun/star/ui/XAcceleratorConfiguration.hpp>
@@ -105,6 +106,7 @@
#include <openuriexternally.hxx>
#include <iostream>
#include <vector>
#include <list>
#include <libxml/xmlwriter.h>
#include <toolkit/awt/vclxmenu.hxx>
#include <unordered_map>
@@ -238,6 +240,17 @@ void SAL_CALL SfxClipboardChangeListener::changedContents( const datatransfer::c
delete pInfo;
}
namespace
{
struct TableSizeType
{
sal_Int32 nRowCount;
sal_Int32 nColCount;
};
}
typedef std::list<uno::Reference<accessibility::XAccessibleTable>> XAccessibleTableList;
namespace
{
@@ -258,6 +271,41 @@ bool isFocused(const accessibility::AccessibleEventObject& aEvent)
return hasState(aEvent, accessibility::AccessibleStateType::FOCUSED);
}
// Put in rAncestorList all ancestors of xTable up to xAncestorTable or
// up to the first not-a-table ancestor if xAncestorTable is not an ancestor.
// xTable is included in the list, xAncestorTable is not included.
// The list is ordered from the ancient ancestor to xTable.
// Return true if xAncestorTable is an ancestor of xTable.
bool getAncestorList(XAccessibleTableList& rAncestorList,
const uno::Reference<accessibility::XAccessibleTable>& xTable,
const uno::Reference<accessibility::XAccessibleTable>& xAncestorTable = uno::Reference<accessibility::XAccessibleTable>())
{
uno::Reference<accessibility::XAccessibleTable> xCurrentTable = xTable;
while (xCurrentTable.is() && xCurrentTable != xAncestorTable)
{
rAncestorList.push_front(xCurrentTable);
uno::Reference<accessibility::XAccessibleContext> xContext(xCurrentTable, uno::UNO_QUERY);
xCurrentTable.clear();
if (xContext.is())
{
uno::Reference<accessibility::XAccessible> xParent = xContext->getAccessibleParent();
uno::Reference<accessibility::XAccessibleContext> xParentContext(xParent, uno::UNO_QUERY);
if (xParentContext.is()
&& xParentContext->getAccessibleRole() == accessibility::AccessibleRole::TABLE_CELL)
{
uno::Reference<accessibility::XAccessible> xCellParent = xParentContext->getAccessibleParent();
if (xCellParent.is())
{
xCurrentTable = uno::Reference<accessibility::XAccessibleTable>(xCellParent, uno::UNO_QUERY);
}
}
}
}
return xCurrentTable.is() && xCurrentTable == xAncestorTable;
}
std::string stateSetToString(::sal_Int64 stateSet)
{
static const std::string states[34] = {
@@ -476,12 +524,33 @@ void aboutParagraph(std::string msg, const uno::Reference<css::accessibility::XA
sal_Int32 nSelectionEnd = xAccText->getSelectionEnd();
aboutParagraph(msg, sText, nCaretPosition, nSelectionStart, nSelectionEnd, force);
}
void aboutFocusedCellChanged(sal_Int32 nOutCount, const std::vector<TableSizeType>& aInList,
sal_Int32 nRow, sal_Int32 nCol, sal_Int32 nRowSpan, sal_Int32 nColSpan)
{
std::stringstream inListStream;
inListStream << "[ ";
for (const auto& rTableSize: aInList)
{
inListStream << "{ rowCount: " << rTableSize.nRowCount << " colCount: " << rTableSize.nColCount << " } ";
}
inListStream << "]";
SAL_INFO("lok.a11y", "LOKDocumentFocusListener::notifyFocusedCellChanged: "
"\n outCount: " << nOutCount
<< "\n inList: " << inListStream.str()
<< "\n row: " << nRow
<< "\n column: " << nCol
<< "\n rowSpan: " << nRowSpan
<< "\n colSpan: " << nColSpan
);
}
} // anonymous namespace
class LOKDocumentFocusListener :
public ::cppu::WeakImplHelper< accessibility::XAccessibleEventListener >
{
static constexpr sal_Int64 MAX_ATTACHABLE_CHILDREN = 30;
static constexpr sal_Int64 MAX_ATTACHABLE_CHILDREN = 100;
const SfxViewShell* m_pViewShell;
std::unordered_set< uno::Reference< uno::XInterface > > m_aRefList;
@@ -489,6 +558,7 @@ class LOKDocumentFocusListener :
sal_Int32 m_nCaretPosition;
sal_Int32 m_nSelectionStart;
sal_Int32 m_nSelectionEnd;
uno::Reference<accessibility::XAccessibleTable> m_xLastTable;
OUString m_sSelectedText;
bool m_bIsEditingCell;
OUString m_sSelectedCellAddress;
@@ -552,11 +622,14 @@ public:
void notifyFocusedParagraphChanged(bool force = false);
void notifyCaretChanged();
void notifyTextSelectionChanged();
void notifyFocusedCellChanged(sal_Int32 nOutCount, const std::vector<TableSizeType>& aInList, sal_Int32 nRow, sal_Int32 nCol, sal_Int32 nRowSpan, sal_Int32 nColSpan);
OUString getFocusedParagraph() const;
int getCaretPosition() const;
private:
void paragraphPropertiesToTree(boost::property_tree::ptree& aPayloadTree, bool force = false) const;
void paragraphPropertiesToJson(std::string& aPayload, bool force = false) const;
bool updateParagraphInfo(const uno::Reference<css::accessibility::XAccessibleText>& xAccText,
bool force, std::string msg = "");
void updateAndNotifyParagraph(const uno::Reference<css::accessibility::XAccessibleText>& xAccText,
@@ -572,21 +645,34 @@ LOKDocumentFocusListener::LOKDocumentFocusListener(const SfxViewShell* pViewShel
{
}
void LOKDocumentFocusListener::paragraphPropertiesToTree(boost::property_tree::ptree& aPayloadTree, bool force) const
{
bool bLeftToRight = m_nCaretPosition == m_nSelectionEnd;
aPayloadTree.put("content", m_sFocusedParagraph.toUtf8().getStr());
aPayloadTree.put("position", m_nCaretPosition);
aPayloadTree.put("start", bLeftToRight ? m_nSelectionStart : m_nSelectionEnd);
aPayloadTree.put("end", bLeftToRight ? m_nSelectionEnd : m_nSelectionStart);
if (force)
aPayloadTree.put("force", 1);
}
void LOKDocumentFocusListener::paragraphPropertiesToJson(std::string& aPayload, bool force) const
{
boost::property_tree::ptree aPayloadTree;
paragraphPropertiesToTree(aPayloadTree, force);
std::stringstream aStream;
boost::property_tree::write_json(aStream, aPayloadTree);
aPayload = aStream.str();
}
OUString LOKDocumentFocusListener::getFocusedParagraph() const
{
aboutView("LOKDocumentFocusListener::getFocusedParagraph", this, m_pViewShell);
aboutParagraph("LOKDocumentFocusListener::getFocusedParagraph",
m_sFocusedParagraph, m_nCaretPosition, m_nSelectionStart, m_nSelectionEnd);
bool bLeftToRight = m_nCaretPosition == m_nSelectionEnd;
boost::property_tree::ptree aPayloadTree;
aPayloadTree.put("content", m_sFocusedParagraph.toUtf8().getStr());
aPayloadTree.put("position", m_nCaretPosition);
aPayloadTree.put("start", bLeftToRight ? m_nSelectionStart : m_nSelectionEnd);
aPayloadTree.put("end", bLeftToRight ? m_nSelectionEnd : m_nSelectionStart);
std::stringstream aStream;
boost::property_tree::write_json(aStream, aPayloadTree);
std::string aPayload = aStream.str();
std::string aPayload;
paragraphPropertiesToJson(aPayload);
OUString sRet = OUString::fromUtf8(aPayload);
return sRet;
}
@@ -620,16 +706,8 @@ int LOKDocumentFocusListener::getCaretPosition() const
void LOKDocumentFocusListener::notifyFocusedParagraphChanged(bool force)
{
aboutView("LOKDocumentFocusListener::notifyFocusedParagraphChanged", this, m_pViewShell);
bool bLeftToRight = m_nCaretPosition == m_nSelectionEnd;
boost::property_tree::ptree aPayloadTree;
aPayloadTree.put("content", m_sFocusedParagraph.toUtf8().getStr());
aPayloadTree.put("position", m_nCaretPosition);
aPayloadTree.put("start", bLeftToRight ? m_nSelectionStart : m_nSelectionEnd);
aPayloadTree.put("end", bLeftToRight ? m_nSelectionEnd : m_nSelectionStart);
aPayloadTree.put("force", force ? 1 : 0);
std::stringstream aStream;
boost::property_tree::write_json(aStream, aPayloadTree);
std::string aPayload = aStream.str();
std::string aPayload;
paragraphPropertiesToJson(aPayload, force);
if (m_pViewShell)
{
aboutParagraph("LOKDocumentFocusListener::notifyFocusedParagraphChanged",
@@ -672,6 +750,59 @@ void LOKDocumentFocusListener::notifyTextSelectionChanged()
}
}
void LOKDocumentFocusListener::notifyFocusedCellChanged(
sal_Int32 nOutCount, const std::vector<TableSizeType>& aInList,
sal_Int32 nRow, sal_Int32 nCol, sal_Int32 nRowSpan, sal_Int32 nColSpan)
{
aboutView("LOKDocumentFocusListener::notifyTablePositionChanged", this, m_pViewShell);
boost::property_tree::ptree aPayloadTree;
if (nOutCount > 0)
{
aPayloadTree.put("outCount", nOutCount);
}
if (aInList.size() > 0)
{
boost::property_tree::ptree aInListNode;
for (const auto& rTableSize: aInList)
{
boost::property_tree::ptree aTableSizeNode;
aTableSizeNode.put("rowCount", rTableSize.nRowCount);
aTableSizeNode.put("colCount", rTableSize.nColCount);
aInListNode.push_back(std::make_pair(std::string(), aTableSizeNode));
}
aPayloadTree.add_child("inList", aInListNode);
}
aPayloadTree.put("row", nRow);
aPayloadTree.put("col", nCol);
if (nRowSpan > 1)
{
aPayloadTree.put("rowSpan", nRowSpan);
}
if (nColSpan > 1)
{
aPayloadTree.put("colSpan", nColSpan);
}
boost::property_tree::ptree aContentNode;
paragraphPropertiesToTree(aContentNode);
aPayloadTree.add_child("paragraph", aContentNode);
std::stringstream aStream;
boost::property_tree::write_json(aStream, aPayloadTree);
std::string aPayload = aStream.str();
if (m_pViewShell)
{
aboutFocusedCellChanged(nOutCount, aInList, nRow, nCol, nRowSpan, nColSpan);
aboutParagraph("LOKDocumentFocusListener::notifyFocusedCellChanged: paragraph: ",
m_sFocusedParagraph, m_nCaretPosition, m_nSelectionStart, m_nSelectionEnd, false);
m_pViewShell->libreOfficeKitViewCallback(LOK_CALLBACK_A11Y_FOCUSED_CELL_CHANGED, aPayload.c_str());
}
}
void LOKDocumentFocusListener::disposing( const lang::EventObject& aEvent )
{
// Unref the object here, but do not remove as listener since the object
@@ -726,7 +857,6 @@ void LOKDocumentFocusListener::updateAndNotifyParagraph(
notifyFocusedParagraphChanged(force);
}
void LOKDocumentFocusListener::notifyEvent(const accessibility::AccessibleEventObject& aEvent )
{
aboutView("LOKDocumentFocusListener::notifyEvent", this, m_pViewShell);
@@ -748,6 +878,8 @@ void LOKDocumentFocusListener::notifyEvent(const accessibility::AccessibleEventO
if( accessibility::AccessibleStateType::FOCUSED == nState )
{
SAL_INFO("lok.a11y", "LOKDocumentFocusListener::notifyEvent: FOCUSED");
uno::Reference<css::accessibility::XAccessibleText> xAccText(xAccessibleObject, uno::UNO_QUERY);
if (m_bIsEditingCell)
{
if (!hasState(aEvent, accessibility::AccessibleStateType::ACTIVE))
@@ -757,8 +889,114 @@ void LOKDocumentFocusListener::notifyEvent(const accessibility::AccessibleEventO
return;
}
}
uno::Reference<css::accessibility::XAccessibleText> xAccText(xAccessibleObject, uno::UNO_QUERY);
updateAndNotifyParagraph(xAccText, false, "STATE_CHANGED: FOCUSED");
// check if we are inside a table: in case notify table and current cell info
bool isInsideTable = false;
uno::Reference<accessibility::XAccessibleContext> xContext(aEvent.Source, uno::UNO_QUERY);
if (xContext.is())
{
uno::Reference<accessibility::XAccessible> xParent = xContext->getAccessibleParent();
if (xParent.is())
{
uno::Reference<accessibility::XAccessibleContext> xParentContext(xParent, uno::UNO_QUERY);
if (xParentContext.is()
&& xParentContext->getAccessibleRole() == accessibility::AccessibleRole::TABLE_CELL)
{
uno::Reference<accessibility::XAccessible> xCellParent = xParentContext->getAccessibleParent();
if (xCellParent.is())
{
uno::Reference<accessibility::XAccessibleTable> xTable(xCellParent, uno::UNO_QUERY);
if (xTable.is())
{
std::vector<TableSizeType> aInList;
sal_Int32 nOutCount = 0;
if (m_xLastTable.is())
{
if (xTable != m_xLastTable)
{
// do we get in one or more nested tables ?
// check if xTable is a descendant of m_xLastTable
XAccessibleTableList newTableAncestorList;
bool isLastAncestorOfNew = getAncestorList(newTableAncestorList, xTable, m_xLastTable);
bool isNewAncestorOfLast = false;
if (!isLastAncestorOfNew)
{
// do we get out of one or more nested tables ?
// check if m_xLastTable is a descendant of xTable
XAccessibleTableList lastTableAncestorList;
isNewAncestorOfLast = getAncestorList(lastTableAncestorList, m_xLastTable, xTable);
// we have to notify "out of table" for all m_xLastTable ancestors up to xTable
// or the first not-a-table ancestor
nOutCount = lastTableAncestorList.size();
}
if (isLastAncestorOfNew || !isNewAncestorOfLast)
{
// we have to notify row/col count for all xTable ancestors starting from the ancestor
// which is a child of m_xLastTable (isLastAncestorOfNew) or the first not-a-table ancestor
for (const auto& ancestor: newTableAncestorList)
{
TableSizeType aTableSize{ancestor->getAccessibleRowCount(),
ancestor->getAccessibleColumnCount()};
aInList.push_back(aTableSize);
}
}
}
}
else
{
// cursor was not inside any table and gets inside one or more tables
// we have to notify row/col count for all xTable ancestors starting from first not-a-table ancestor
XAccessibleTableList newTableAncestorList;
getAncestorList(newTableAncestorList, xTable);
for (const auto& ancestor: newTableAncestorList)
{
TableSizeType aTableSize{ancestor->getAccessibleRowCount(),
ancestor->getAccessibleColumnCount()};
aInList.push_back(aTableSize);
}
}
// we have to notify current row/col of xTable and related row/col span
sal_Int64 nChildIndex = xParentContext->getAccessibleIndexInParent();
sal_Int32 nRow = xTable->getAccessibleRow(nChildIndex);
sal_Int32 nCol = xTable->getAccessibleColumn(nChildIndex);
sal_Int32 nRowSpan = xTable->getAccessibleRowExtentAt(nRow, nCol);
sal_Int32 nColSpan = xTable->getAccessibleColumnExtentAt(nRow, nCol);
m_xLastTable = xTable;
updateParagraphInfo(xAccText, false, "STATE_CHANGED: FOCUSED");
notifyFocusedCellChanged(nOutCount, aInList, nRow, nCol, nRowSpan, nColSpan);
isInsideTable = true;
}
}
}
}
}
// paragraph is not inside any table
if (!isInsideTable)
{
if (m_xLastTable.is())
{
// we get out one or more tables
// we have to notify "out of table" for all m_xLastTable ancestors
// up to the first not-a-table ancestor
XAccessibleTableList lastTableAncestorList;
getAncestorList(lastTableAncestorList, m_xLastTable);
sal_Int32 nOutCount = lastTableAncestorList.size();
// no more inside a table
m_xLastTable.clear();
// notify
std::vector<TableSizeType> aInList;
updateParagraphInfo(xAccText, false, "STATE_CHANGED: FOCUSED");
notifyFocusedCellChanged(nOutCount, aInList, -1, -1, 1, 1);
}
else
{
updateAndNotifyParagraph(xAccText, false, "STATE_CHANGED: FOCUSED");
}
}
aboutTextFormatting("LOKDocumentFocusListener::notifyEvent: STATE_CHANGED: FOCUSED", xAccText);
}
break;