基于Flaui的xpath,实现winui自动化操作
By
瞌睡蟲子
at 2023-02-02 • 1人收藏 • 2242人看过
aardio自带了MSAA,UIA的相关库,MSAA库还好,UIA的库写起来太麻烦了,又懒得封装。于是找找现成的库直接玩玩
测试代码:
import console; import dotNet; import mouse; var Core = dotNet.load("FlaUI.Core","D:\Program Files (x86)\flaUI\Libs\FlaUI.Core.dll"); var UIA2 = dotNet.load("FlaUI.UIA2","D:\Program Files (x86)\flaUI\Libs\FlaUI.UIA2.dll"); var UIA3 = dotNet.load("FlaUI.UIA3","D:\Program Files (x86)\flaUI\Libs\FlaUI.UIA3.dll"); var Interop = dotNet.load("Interop.UIAutomationClient","D:\Program Files (x86)\flaUI\Libs\Interop.UIAutomationClient.dll"); Application = Core.import("FlaUI.Core.Application"); AutomationType = UIA3.new("FlaUI.UIA3.UIA3Automation"); TimeSpan = dotNet.import("System.TimeSpan","mscorlib.dll") var app = Application.Attach("WeChat.exe"); console.log(app) var window = app.GetMainWindow(AutomationType,TimeSpan.FromSeconds(2.0)); if(window){ console.log(window.Title) var ele = window.FindFirstByXPath("/Pane[3]/Pane[2]/Pane[3]/Pane/Pane/Pane/Pane/Pane[2]/Pane[2]/Pane[2]/Pane/Pane[1]/Edit") //window.FindAllByXPath("") console.log(ele) console.log(ele.BoundingRectangle) window.SetForeground() var rect = ele.BoundingRectangle; x,y = rect.x + rect.Width/2 , rect.y + rect.Height/2 mouse.click(x,y,true) } console.pause()
运行效果
dll直接从工具里面copy过来:
14 个回复 | 最后更新于 2024-01-28
封装了wpath库,方便调用
WPath.aardio
import dotNet import System.Windows.Automation if(table.getByNamespace("WPath")){ return; } var assembly = dotNet.load("WPath",$"/res/WPath.dll"); assembly.import("WPath.UiaExtension"); root = System.Windows.Automation.AutomationElement.RootElement; function wpath(xpath,ele){ ele := root return WPath.UiaExtension.FindByWPath(ele,xpath); } /**intellisense() WPath.UiaExtension = winui元素WPath查找\n WPath.UiaExtension.FindByWPath(.(指定AutomationElement元素,xpath表达式) = WPath查找winui元素,默认桌面元素root。\n!autoEle. wpath(.(xpath,指定AutomationElement元素) = WPath查找winui元素,默认桌面元素root。 end intellisense**/
调用代码
import WPath; import console; var ee = wpath("/window[@Name='微信']//edit[@Name='输入']"); console.log(ee.Current.BoundingRectangle ) console.pause()
flaui从桌面开始查找
import console; import dotNet; import mouse; var assembly = dotNet.load("FlaUI",$"/res/FlaUI.dll"); assembly.import("FlaUI.UIA3"); root = FlaUI.UIA3.UIA3Automation().GetDesktop(); var tm = time.tick(); var ele = root.FindFirstByXPath("/Window[@Name='微信']//Edit[@Name='输入']") console.log(time.tick()-tm); console.log(ele) console.log(ele.BoundingRectangle) ele.SetForeground() sleep(3000) var rect = ele.BoundingRectangle; x,y = rect.x + rect.Width/2 , rect.y + rect.Height/2 mouse.click(x,y,true) console.pause()
补充一个flaui自用库
import dotNet; import mouse; import key; import console; namespace myplu.flaui; var assembly = ..dotNet.load("FlaUI",$"~\lib\myplu\flaui\.res\FlaUI.dll"); TimeSpan = ..dotNet.import("System.TimeSpan","mscorlib"); assembly.import("FlaUI.Core"); assembly.import("FlaUI.UIA3"); application = ..FlaUI.Core.Application; auto = ..FlaUI.UIA3.UIA3Automation(); input = null; automationElementExtensions = null; dbg = null; root = auto.GetDesktop(); registerFocusChanged = function(callback){ return auto.RegisterFocusChangedEvent(callback); } unRegisterFocusChanged = function(eventHandler){ auto.UnregisterFocusChangedEvent(eventHandler); } pointElement = function(){ if(!input){ assembly.import("FlaUI.Core.Input"); input = ..FlaUI.Core.Input.Mouse; } return auto.FromPoint(input.Position); } elementHighlight = function(element){ if(!automationElementExtensions){ assembly.import("FlaUI.Core.AutomationElements"); automationElementExtensions = ..FlaUI.Core.AutomationElements.AutomationElementExtensions; } automationElementExtensions.DrawHighlight(element); } getXPathToElement = function(element, rootElement){ if(!dbg){ dbg = ..FlaUI.Core.Debug; } return dbg.GetXPathToElement(element,rootElement); } attach = function(exeName){ var app = application.Attach(exeName); return app.GetMainWindow(auto,TimeSpan.FromSeconds(2.0)); } findFirstByXPath = function(expression, element){ if(expression){ if(element){ return element.FindFirstByXPath(expression); }else { return root.FindFirstByXPath(expression); } } } findAllByXPath = function(expression, element){ if(expression){ if(element){ return element.FindAllByXPath(expression); }else { return root.FindAllByXPath(expression); } } } clickElement = function(expression, element, clickType, moveMouse, xScale=0.5, yScale=0.5){ var ele = findFirstByXPath(expression, element); if(ele){ //ele.Focus(); ele.SetForeground(); select(clickType) { case "L" { click(ele,moveMouse, xScale, yScale); } case "LD" { doubleClick(ele,moveMouse, xScale, yScale); } case "R" { rightClick(ele,moveMouse, xScale, yScale); } case "RD" { rightDoubleClick(ele,moveMouse, xScale, yScale); } else { click(ele,moveMouse, xScale, yScale); } } } return ele; } sendElementString = function(expression, element, str, clickType, moveMouse, xScale=0.5, yScale=0.5){ var ele = clickElement(expression, element, clickType, moveMouse, xScale, yScale); if(ele){ ..key.sendString(str); } return ele; } var getControlPoint = function(automationElement, xScale=0.5, yScale=0.5){ var rect = automationElement.BoundingRectangle; return rect.x + rect.Width*xScale , rect.y + rect.Height*yScale } click = function(automationElement,moveMouse, xScale=0.5, yScale=0.5){ var x,y = getControlPoint(automationElement, xScale, yScale); if(moveMouse){ ..mouse.moveTo(x,y,true); } ..mouse.click(x,y,true); } doubleClick = function(automationElement,moveMouse, xScale=0.5, yScale=0.5){ var x,y = getControlPoint(automationElement, xScale, yScale); if(moveMouse){ ..mouse.moveTo(x,y,true); } ..mouse.clickDb(x,y,true); } rightClick = function(automationElement,moveMouse, xScale=0.5, yScale=0.5){ var x,y = getControlPoint(automationElement, xScale, yScale); if(moveMouse){ ..mouse.moveTo(x,y,true); } ..mouse.rb.click(x,y,true); } rightDoubleClick = function(automationElement,moveMouse, xScale=0.5, yScale=0.5){ var x,y = getControlPoint(automationElement, xScale, yScale); if(moveMouse){ ..mouse.moveTo(x,y,true); } ..mouse.rb.click(x,y,true); } destory = function(){ application = null; auto = null; root = null; collectgarbage("collect"); } each = ..dotNet.each /*****intellisense() myplu.flaui = 窗口自动化工具,基于flaui myplu.flaui.application = FlaUI.Core.Application构建对象 myplu.flaui.auto = FlaUI.UIA3构建对象\n!flaUI3Obj. myplu.flaui.root = 桌面元素,根元素\n!automationElement. myplu.flaui.attach("__") = 指定进程名或者路径,获取主窗口元素\n!automationElement. myplu.flaui.findFirstByXPath(.(xpath表达式,查找元素) = 以xpath方式在指定元素查找,返回第一个匹配到的元素\n@2可选\n!automationElement. myplu.flaui.findAllByXPath(.(xpath表达式,查找元素) = 以xpath方式在指定元素查找,返回所有匹配到的元素\n@2可选\n!automationElementList. myplu.flaui.clickElement(.(xpath表达式,查找元素,点击类型【L/LD/R/RD】,是否移动,x坐标偏移比例,y坐标偏移比例) = 以xpath方式在指定元素查找,激活并点击元素,返回所有匹配到的元素\n@1必填\n!automationElementList. myplu.flaui.sendElementString(.(xpath表达式,查找元素,发送文字,点击类型【L/LD/R/RD】,是否移动,x坐标偏移比例,y坐标偏移比例) = 以xpath方式在指定元素查找,激活并点击元素,然后发送文本。返回所有匹配到的元素\n@1、@2必填\n!automationElementList. myplu.flaui.click(.(automationElement,是否移动,x坐标偏移比例,y坐标偏移比例) = 控件点击,前提确认控件是否支持点击。默认不显示鼠标移动轨迹 myplu.flaui.doubleClick(.(automationElement,是否移动,x坐标偏移比例,y坐标偏移比例) = 控件双击,前提确认控件是否支持点击。默认不显示鼠标移动轨迹 myplu.flaui.rightClick(.(automationElement,是否移动,x坐标偏移比例,y坐标偏移比例) = 右键单机,默认不显示鼠标移动轨迹 myplu.flaui.rightDoubleClick(.(automationElement,是否移动,x坐标偏移比例,y坐标偏移比例) = 右键双击,默认不显示鼠标移动轨迹 myplu.flaui.each(automationElementList) = @for i,v in ??.each(__/*输入需要遍历元素数组,\n返回值 i 为当前索引,v 为当前值,\n注意并非所有 .NET 类型都支持此接口*/) { } myplu.flaui.registerFocusChanged() = @var eventHandler = myplu.flaui.registerFocusChanged(function(element){ /*注册监听,焦点变更的元素*/ }); myplu.flaui.unRegisterFocusChanged(eventHandler) = 解除焦点变更监听 myplu.flaui.pointElement() = 获取鼠标指向元素,不一定获取到,通过焦点变更获取 myplu.flaui.elementHighlight(__) = 元素高亮 myplu.flaui.getXPathToElement(.(目标元素,起始元素) = 获取从起始元素到目标元素的正则表达式 myplu.flaui.destory() = 释放资源 end intellisense*****/ /*****intellisense(!flaUI3Obj) GetDesktop() = 获取桌面元素,根元素\n!automationElement. Compare(.(automationElement1,automationElement2) = 判断是否同一个AutomationElement元素 FocusedElement() = 获取当前焦点元素\n!automationElement. FromHandle(hwnd) = 获取指定控件句柄元素\n!automationElement. FromPoint(point) = 获取鼠标指向控件元素\n!automationElement. end intellisense*****/ /*****intellisense(!automationElement) ActualHeight = 显示区域高度 ActualWidth = 显示区域宽度 AutomationId = 控件ID BoundingRectangle = 控件显示区域 CachedChildren = 缓存子元素\n!automationElementList. CachedParent = 缓存父元素\n!automationElement. ClassName = 类名 ControlType = 控件类型 HelpText = 帮助文档 IsEnabled = 是否激活 Name = 控件标题 Parent = 父元素\n!automationElement. Capture() = 截图,返回System.Drawing.Bitmap CaptureToFile(.(filePath) = 截图保存为文件 Click(.(moveMouse) = 控件点击,前提确认控件是否支持点击。默认不显示鼠标移动轨迹 DoubleClick(.(moveMouse) = 控件双击,前提确认控件是否支持点击。默认不显示鼠标移动轨迹 FindAll(.(treeScope, condition) = 查找元素,指定搜索方式,表达式查找。\n!automationElementList. FindAllByXPath(.(xPath) = xpath查找元素\n!automationElementList. FindAllChildren() = 查找所有子元素\n!automationElementList. FindAllDescendants() = 查找所有后辈元素\n!automationElementList. FindAllWithOptions(.(treeScope, condition, traversalOptions, root) = 条件查找\n!automationElementList. FindAt(.(treeScope, index, condition) = 下标查找,指定搜索方式,表达式查找。\n!automationElement. FindChildAt(.(index, condition) = 下标查找查找子元素\n!automationElement. FindFirst(.(treeScope, condition) = 查找元素,返回第一个匹配到的元素。\n!automationElement. FindFirstByXPath(.(xPath) = xpath查找元素,返回第一个匹配到的元素。\n!automationElement. FindFirstChild() = 查找第一个子元素\n!automationElement. FindFirstChild(.(automationId) = 查找第一个子元素,通过控件ID。\n!automationElement. FindFirstChild(.(condition) = 查找第一个子元素,表达式查找。\n!automationElement. FindFirstDescendant() = 查找第一个后辈元素\n!automationElement. FindFirstDescendant(.(automationId) = 查找第一个后辈元素,通过控件ID。\n!automationElement. FindFirstDescendant(.(condition) = 查找第一个后辈元素,表达式查找。\n!automationElement. FindFirstWithOptions(.(treeScope, condition, traversalOptions, root) = 条件查找\n!automationElement. Focus() = 设置焦点 GetClickablePoint() = 获取鼠标位置 GetCurrentMetadataValue(.(targetId, metadataId) = 获取Metadata值 RightClick(.(moveMouse) = 右键单机,默认不显示鼠标移动轨迹 RightDoubleClick(.(moveMouse) = 右键双击,默认不显示鼠标移动轨迹 SetForeground() = 激活 end intellisense*****/ /*****intellisense(!automationElementList) each() = @for i,v in myplu.flaui.each(__/*输入需要遍历的 .NET 对象或普通数组,\n返回值 i 为当前索引,v 为当前值,\n注意并非所有 .NET 类型都支持此接口*/) { } end intellisense*****/
补充一个flaui元素拾取工具例子
// flaui元素拾取测试 io.open() io.print("正在监视按键,按ESC键退出!") //导入HOOK库 import key.hook import com; import win; import winex; import myplu.flaui; import dotNet //创建录制钩子 hk = key.hook(); var eventHandler = myplu.flaui.registerFocusChanged(function(element){ /*注册监听,焦点变更的元素*/ myplu.flaui.elementHighlight(element); io.print(myplu.flaui.getXPathToElement(element)); io.print(elememt.ControlType); }) //录制回调函数 hk.proc=function(msg,vkcode,scancode){ var kn = key.getName( vkcode ); if( ( vkcode == key.VK.LCTRL) && ( msg == 0x100/*_WM_KEYDOWN*/ ) ){ ::PostThreadMessage(thread.getId(),1234,0,0) //win.quitMessage(); } } //录制需要消消息循环 import win; win.loopMessage( function(msg){ if(msg.message==1234){ var ele = myplu.flaui.pointElement(); myplu.flaui.elementHighlight(ele); io.print(myplu.flaui.getXPathToElement(ele)); io.print(ele.ControlType); } } ) //一定要关闭钩子 hk.close(); myplu.flaui.unRegisterFocusChanged(eventHandler); execute("pause") //按任意键继续 io.close();//关闭控制台
微信自动化+嗅探数据抓包例子
import myplu.flaui; import process; import fsys.lnk; import myplu.fiddler; import win; import console; import key; var path,param = fsys.lnk.search("WeChat.exe"); process.execute(path,param) //sleep(3000); var ele = myplu.flaui.attach(path); myplu.flaui.clickElement("/Pane/Pane/ToolBar/Button[@Name='通讯录']",ele); sleep(500) myplu.flaui.clickElement("/Pane/Pane/ToolBar/Button[@Name='聊天']",ele); sleep(500) myplu.flaui.sendElementString("/Pane/Pane/Pane/Pane/Pane/Pane/Edit[@Name='搜索']",ele,"文件传输助手"); sleep(1000) key.press(0xD/*_VK_ENTER*/); sleep(500) myplu.flaui.sendElementString("/Pane/Pane/Pane/Pane/Pane/Pane/Pane/Pane/Pane/Pane/Pane/Pane/Edit[@Name='文件传输助手']",ele,"https://aspmo2o.mercedes-benz.com.cn/sm/#/"); sleep(500) key.press(0xD/*_VK_ENTER*/); sleep(500) myplu.flaui.clickElement("/Pane/Pane/Pane/Pane/Pane/Pane/Pane/Pane/Pane/Pane/List[@Name='消息']/ListItem[last()]",ele,"L",false,0.5,0.3); sleep(3000) var mesgs = {}; var isOk; /* 接受消息前事件*/ myplu.fiddler.fiddlerApplication.BeforeResponse = function (s) { if(string.find(s.fullUrl,"@https://aspmo2o.mercedes-benz.com.cn/v2/basis/getAllProvinces") or string.find(s.fullUrl,"@https://aspmo2o.mercedes-benz.com.cn/v2/basis/vehiclePickerData")){ table.push(mesgs,{url=s.fullUrl; method=s.RequestMethod; responseCode=s.responseCode; requestHeaders=myplu.fiddler.getHeaders(s.RequestHeaders); responseHeaders=myplu.fiddler.getHeaders(s.ResponseHeaders); requestBody=s.GetRequestBodyAsString(); responseBody=s.GetResponseBodyAsString();}); } if(#mesgs == 2){ isOk = true; ::PostThreadMessage(thread.getId(),1234,0,0); } } /* 启动嗅探进程,设置代理端口9999*/ myplu.fiddler.startup(9999); ele = myplu.flaui.findFirstByXPath("/Pane[@Name='微信']"); console.log(ele); if(ele){ var e1; do{ sleep(500); e1 = ele.FindFirstByXPath("/Document/Spinner[@AutomationId='phoneNumber']"); }while(tostring(e1)); console.log(ele); e1.Focus(); key.sendString("xxx",100); } /** var sh = myplu.flaui.findFirstByXPath("/Document/Button[@Name='稍后绑车']",ele); if(sh){ myplu.flaui.clickElement("/Document/Button[@Name='稍后绑车']",ele); sleep(2000); } myplu.flaui.clickElement("/Document/List/ListItem/Text[@Name='轮胎']",ele); **/ do{ win.pumpMessage(); }while(!isOk) myplu.fiddler.stop(); for(k,v in mesgs){ console.dumpTable(v); }
登录后方可回帖
补充一个WPath实现的xpath
demo:
dll下载地址:
https://github.com/tobyqin/wpath/releases/tag/v1.0