Android 辅助服务实战-游戏点击器

背景：

前几年我一直在玩一款氪金养成类手游<<末日危机>>，每天都有任务需要完成，那时游戏里面还没有一键收菜，我去年（接手了公会里面不少弃坑号，越玩越是耗时间）就想做了一些辅助软件减轻我的负担，先是在PC做了个自动点击完成关卡战斗，后面觉得天天开虚拟机太麻烦，不如直接在手机上完成。

思路：

Android系统有提供一个无障碍功能：AccessbilityService，在Android6.0之前能访问第三方应用的当前布局和控件结点，现在Android手机系统等级动辄10以上，已经不能用这种直接获取控件的方式，要使用截图再进行图片识别目标位置，再进行坐标位置上的点击。

首先手机上需要开启辅助服务，该服务在手机截图API实现要求手机版本至少要Android10。具体代码如下

private fun isAccessibilitySettingsOn(mContext: Context): Boolean {var accessibilityEnabled = 0// TestService为对应的服务val service = packageName + "/" + AccessbilityServiceImp::class.java.canonicalNameLog.i(TAG, "service:$service")try {accessibilityEnabled = Settings.Secure.getInt(mContext.applicationContext.contentResolver,Settings.Secure.ACCESSIBILITY_ENABLED)Log.v(TAG, "accessibilityEnabled = $accessibilityEnabled")} catch (e: SettingNotFoundException) {Log.e(TAG, "Error finding setting, default accessibility to not found: " + e.message)}val mStringColonSplitter = SimpleStringSplitter(':')if (accessibilityEnabled == 1) {Log.v(TAG, "***ACCESSIBILITY IS ENABLED*** -----------------")val settingValue = Settings.Secure.getString(mContext.applicationContext.contentResolver,Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES)if (settingValue != null) {mStringColonSplitter.setString(settingValue)while (mStringColonSplitter.hasNext()) {val accessibilityService = mStringColonSplitter.next()Log.v(TAG,"-------------- > accessibilityService :: $accessibilityService $service")if (accessibilityService.equals(service, ignoreCase = true)) {Log.v(TAG,"We've found the correct setting - accessibility is switched on!")return true}}}} else {Log.v(TAG, "***ACCESSIBILITY IS DISABLED***")}return false
}

具体实现：

开启了辅助服务后，我们需要一个媒介和该服务进行交互，并且不能影响后续的截图效果，一种能在屏幕上进行收缩的工具栏。

我采用系统消息通知栏+广播来解决这个问题，它会和辅助服务进行交互：

object GameNotification {private var manager: NotificationManager? = nullprivate var remoteViews: RemoteViews? = nullvar notify: Notification? = nullfun createNotificaton(context: Context) {manager = context.getSystemService(AccessibilityService.NOTIFICATION_SERVICE) as NotificationManager// 设置通知栏的图片文字remoteViews = RemoteViews(context.packageName,R.layout.custom_notice)remoteViews!!.setImageViewResource(R.id.widget_play, R.drawable.ic_baseline_play_arrow_24)remoteViews!!.setImageViewResource(R.id.widget_stop, R.drawable.ic_baseline_stop_24)val builder = NotificationCompat.Builder(context)val intent = Intent(context, MainActivity::class.java)// 点击跳转到主界面val intent_go = PendingIntent.getActivity(context, 0, intent,PendingIntent.FLAG_UPDATE_CURRENT)remoteViews!!.setOnClickPendingIntent(R.id.notice, intent_go)// 设置开始按键val start = Intent()start.action = Command.ACTION_STARTval intent_start = PendingIntent.getBroadcast(context, 0, start,PendingIntent.FLAG_UPDATE_CURRENT)remoteViews!!.setOnClickPendingIntent(R.id.widget_play, intent_start)// 设置停止按键val stop = Intent()stop.action = Command.ACTION_STOPval intent_stop = PendingIntent.getBroadcast(context, 1, stop,PendingIntent.FLAG_UPDATE_CURRENT)remoteViews!!.setOnClickPendingIntent(R.id.widget_stop, intent_stop)builder.setSmallIcon(R.drawable.ic_launcher_background) // 设置顶部图标if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {val mChannel = NotificationChannel("123", "147", NotificationManager.IMPORTANCE_LOW)manager!!.createNotificationChannel(mChannel)notify = Notification.Builder(context).setChannelId("123")//                    .setContentTitle("5 new messages")//                    .setContentText("hahaha").setContent(remoteViews).setSmallIcon(R.mipmap.ic_launcher).build()} else {notify = builder.build()notify!!.contentView = remoteViews // 设置下拉图标notify!!.bigContentView = remoteViews // 防止显示不完全,需要添加apisupportnotify!!.flags = Notification.FLAG_ONGOING_EVENT}manager!!.notify(100, notify)}}

通过广播进行消息传输，告诉辅助服务可以执行或者停止当前选择的任务。

识别图片：

屏幕截图获取代码片段：

执行截图：

@RequiresApi(Build.VERSION_CODES.R)
fun startSnapShoot() {accessbilityServiceImp!!.mHandler.postDelayed({accessbilityServiceImp!!.takeScreenshot(Display.DEFAULT_DISPLAY,accessbilityServiceImp!!.mainExecutor,mCallBack)},1000)
}

接收截图数据：

 @RequiresApi(Build.VERSION_CODES.R)var mCallBack:AccessibilityService.TakeScreenshotCallback = object :AccessibilityService.TakeScreenshotCallback{override fun onSuccess(screenshotResult: AccessibilityService.ScreenshotResult) {val nodeInfo = accessbilityServiceImp?.rootInActiveWindowval result = nodeInfo?.let { accessbilityServiceImp?.findNodeInfosByText(it,"任务") }Log.e("AccessbilityServiceImp","onSuccess = "+screenshotResult+",result = "+result)var timespec = 500LaccessbilityServiceImp!!.timespec = 1000Lval hardwareBuffer = screenshotResult.hardwareBufferval colorSpace = screenshotResult.colorSpaceif (hardwareBuffer.width > 0 && hardwareBuffer.height > 0 && colorSpace != null) {val bitmap = Bitmap.wrapHardwareBuffer(hardwareBuffer, colorSpace)...}}}...

把获取的数据转换成bitmap,后面使用opencv进行图片识别。

竞技场挑战任务：

竞技场页面获取当前在场队伍的战力值：

 fun detectNumberRect(bitmap: Bitmap, list: List<Mat>): List<ComplyPlayInfo> {val bitmapNew: Bitmap = bitmap.copy(Bitmap.Config.ARGB_8888, true)var resultPlays :List<ComplyPlayInfo>val dectectedPlays = ArrayList<ComplyPlayInfo>()var myPlayInfo = ComplyPlayInfo(0,0)//opencv初始化if (!OpenCVLoader.initDebug()) {Log.d("--------opencv--------", "OpenCVLoader error")}//Mat変換val src = Mat(bitmapNew.height, bitmapNew.width, CvType.CV_8UC4)Utils.bitmapToMat(bitmapNew, src)//获取战力值所在的坐标位置集合val rects = detectPowerNumberRect(bitmap)rects.mapIndexed {index,it->val mRgb = Mat(src, it)val b = Bitmap.createBitmap(mRgb!!.cols(), mRgb.rows(),Bitmap.Config.ARGB_8888)Utils.matToBitmap(mRgb, b)val dst = Mat.zeros(Size(src.width().toDouble(), src.height().toDouble()), CvType.CV_8UC3)//绘制轮廓 (Yellow)var color = Scalar(255.0, 255.0, 0.0)//灰色化Imgproc.cvtColor(mRgb, mRgb, Imgproc.COLOR_RGB2GRAY)bitwise_not(mRgb, mRgb);//二値化Imgproc.threshold(mRgb, mRgb, 0.0, 250.0,Imgproc.THRESH_BINARY or Imgproc.THRESH_OTSU)Utils.matToBitmap(mRgb, b)//比特反转val hierarchy = Mat.zeros(Size(0.0, 0.0), CvType.CV_8UC1)// 輪郭抽出val contours: List<MatOfPoint> = ArrayList()Imgproc.findContours(mRgb,contours,hierarchy,Imgproc.RETR_TREE,Imgproc.CHAIN_APPROX_SIMPLE)Imgproc.drawContours(dst, contours, -1, color, 1)var box: Rect?var boxs = ArrayList<Rect>()for (i in contours.indices) {val ptmat: MatOfPoint = contours[i]// 轮廓的重心的绘制 (Red)color = Scalar(255.0, 0.0, 0.0)val ptmat2 = MatOfPoint2f(*ptmat.toArray())val bbox = Imgproc.minAreaRect(ptmat2)box = bbox.boundingRect()if (box.width < box.height && box.height < 50 && box.height > 25&& box.y < 10) {Imgproc.circle(dst, bbox.center, 5, color, -1)// 周围轮廓四角形绘制 (Green)color = Scalar(0.0, 255.0, 0.0)Imgproc.rectangle(dst, box.tl(), box.br(), color, 2)boxs.add(box)}}boxs.sortBy {it.x}val result = ArrayList<NumberInfo>()var num = 0Lboxs.mapIndexed { index, it ->if (it.x+it.width>mRgb.width()){it.width = mRgb.width()-it.x}val tmp = Mat(mRgb, it)val tmpBitmap = Bitmap.createBitmap(tmp!!.cols(), tmp.rows(),Bitmap.Config.ARGB_8888)Utils.matToBitmap(tmp, tmpBitmap)list.mapIndexed { index, it ->val result_cols: Int = abs(tmp.cols() - it.cols()) + 1val result_rows: Int = abs(tmp.rows() - it.rows()) + 1val res = Mat(result_rows, result_cols, CvType.CV_32FC1)//归一化var item = itval bitmap2 = Bitmap.createBitmap(it!!.cols(), it!!.rows(),Bitmap.Config.ARGB_8888)Utils.matToBitmap(it, bitmap2)Imgproc.resize(it, it, Size(tmp.width().toDouble(), tmp.height().toDouble()))Imgproc.matchTemplate(tmp, item, res, Imgproc.TM_SQDIFF)val bitmap = Bitmap.createBitmap(tmp!!.cols(), tmp!!.rows(),Bitmap.Config.ARGB_8888)Utils.matToBitmap(tmp, bitmap)val bitmap1 = Bitmap.createBitmap(item!!.cols(), item!!.rows(),Bitmap.Config.ARGB_8888)Utils.matToBitmap(item, bitmap1)//获得最可能点，MinMaxLocResult是其数据格式，包括了最大、最小点的位置x、yval mlr = Core.minMaxLoc(res)result.add(NumberInfo(mlr.minVal, index))Log.e("AccessbilityServiceImp", "mlr result minVal="+ mlr.minVal + ",maxVal=" + mlr.maxVal + ",minLoc=" + mlr.minLoc + ",maxLoc=" + mlr.maxLoc)}result.sortBy {it.minValue}num += result[0].index * 10.0.pow(boxs.size.toDouble() - 1 - index).toLong()result.clear()}if (index == 0){myPlayInfo = ComplyPlayInfo(num,index)}else{dectectedPlays.add(ComplyPlayInfo(num,index))}Log.e("AccessbilityServiceImp", "num result =" + num)}resultPlays = dectectedPlays.filter { myPlayInfo.strength > it.strength }Log.e("AccessbilityServiceImp", "resultPlays =" + resultPlays)return resultPlays}

调用 Imgproc.matchTemplate()，使用数字模板对比，相似度高的就认为是该阿拉伯数字，逐个数字叠加后获得战力值，返回战力值比自己队伍低的集合，上层获取到该值后挑选还能挑战的队伍进行战斗，（相同的队伍当天只能挑战它8次）。

任务栏刷任务：

在任务栏进行刷任务操作需要进行的步骤：识别当前视图所有的任务的星数（1到7星）->挑选符合条件的任务刷新->向右滑动任务->重新识别还没有操作过的任务->刷新完后点击一键领取已经完成的任务。

识别当前任务的星数代码片段：

    private fun cutStarsPictrue(bitmap: Bitmap, list: ArrayList<Rect>): List<TaskInfo> {val bitmapNew: Bitmap = bitmap.copy(Bitmap.Config.ARGB_8888, true)//opencv初始化if (!OpenCVLoader.initDebug()) {Log.d("--------opencv--------", "OpenCVLoader error")}//Mat变换val mRgb = Mat(bitmapNew.height, bitmapNew.width, CvType.CV_8UC4)Utils.bitmapToMat(bitmapNew, mRgb)var starsNum = ArrayList<TaskInfo>()list.map {val rect = Rect(it.x + 15, it.y + 8, it.width - 35, it.height - 18)val src = Mat(mRgb, rect)val b = Bitmap.createBitmap(src!!.cols(), src.rows(),Bitmap.Config.ARGB_8888)Utils.matToBitmap(src, b)val num = returnStarsNum(b)starsNum.add(TaskInfo(rect,num))Log.e("AccessbilityServiceImp", " num =" + num)}return starsNum}

识别当前任务的内容代码片段：（主要是识别当前任务是否符合条件，三星任务只需要英雄碎片，4星任务抛弃竞技场挑战券，4星以上任务全部完成）

fun cutTaskContent(bitmap: Bitmap, list: List<TaskInfo>): List<TaskInfo> {val bitmapNew: Bitmap = bitmap.copy(Bitmap.Config.ARGB_8888, true)//opencv初始化if (!OpenCVLoader.initDebug()) {Log.d("--------opencv--------", "OpenCVLoader error")}//Mat変換val mRgb = Mat(bitmapNew.height, bitmapNew.width, CvType.CV_8UC4)Utils.bitmapToMat(bitmapNew, mRgb)var starsNum = ArrayList<TaskInfo>()var rect:Rectlist.map {//如果是3星任务截图区域if (it.stars < 4){rect = Rect(it.rect.x + bitmapNew.width/23, it.rect.y - 3 + bitmapNew.height/32*9, it.rect.width/3 , it.rect.height/2 )}else {//如果是4-5星任务截图区域rect = Rect(it.rect.x + bitmapNew.width / 60,it.rect.y + bitmapNew.height / 33 * 6,it.rect.width / 5,it.rect.width / 3)}val src = Mat(mRgb, rect)val b = Bitmap.createBitmap(src!!.cols(), src.rows(),Bitmap.Config.ARGB_8888)Utils.matToBitmap(src, b)val num = returnTaskStarsNum(b)/*如果是3星任务返回5是英雄碎片如果是4，5星任务返回5是竞技场挑战券*/if (it.stars < 4) {if (num >= 4) {starsNum.add(it)}}else{if (num in 5..9) {starsNum.add(it)}}Log.e("AccessbilityServiceImp", " starsNum =" + starsNum)}return starsNum
}

指令执行：

复杂的点击和滑动指令操作需要一个执行者来完成：Handler

val mHandler:TaskHandler = TaskHandler()class TaskHandler:Handler(Looper.getMainLooper()){override fun handleMessage(msg: Message) {val callback = msg.obj as ()->Unitcallback.invoke()}}

用消息把不同的指令传给Handler去处理，可以延时执行，也可以随时取消没有完成的指令任务。

    //滑动fun slidingScreen(scrollData: ScrollData, delayTime: Long){val callback = {if (height != 0f) {if (width != 0f) {ActionDeal(width/2,height/2,scrollData,62)}}}timespec = delayTimemHandler.postDelayed(callback,-1,delayTime)}//点击fun clickScreen(x:Float,y: Float){val callback = {ActionDeal(x,y,null,62)}mHandler.postDelayed(callback,-1,timespec)timespec+=1000Log.e("AccessbilityServiceImp","timespec = "+timespec)}//点击（带时间差）fun clickScreen(x:Float,y: Float,delayTime: Long){val callback = {if (height != 0f) {if (width != 0f) {ActionDeal(x,y,null,62)}}}mHandler.postDelayed(callback,62,delayTime)}

不用指令之间的等待时间各不相同，需要给根据实际的时间差来完成一连串指令。

创建任务实体类：

data class TaskInfo(val callback: () -> Unit, val timespec: Long, val id:Int)

创建任务集合：

var taskList = ArrayList<TaskInfo>()

Handler执行所有的指令任务：

taskList.add(TaskInfo({localBroadcastReceiver.startSnapShoot()},timespec,48))taskList.map {Log.e("AccessbilityServiceImp","timespec = "+it.timespec)mHandler.postDelayed(it.callback,it.timespec)
}

传送门：项目源码地址
创造不易，欢迎点赞starred me