commit d17080bcc8e3b6cb4aedcd8fcb4120570caf4e49 Author: rudals252 Date: Mon Jan 26 10:48:13 2026 +0900 초기 업로드: OnnxManager 리소스 해제 코드 적용 및 메모리 누수 방지 패치 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..10cfdbf --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +*.iml +.gradle +/local.properties +/.idea +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/README.md b/README.md new file mode 100644 index 0000000..1c6b02d --- /dev/null +++ b/README.md @@ -0,0 +1,55 @@ +# Mobility Gateway for Android + +Kepco AI 엔진의 기본적인 detect 기능을 안드로이드 앱으로 구현 +onnx export를 통해 YOLOv8 object detect, pose estimate 동작 확인 +REST 서버 동작 확인 및 MQTT 통신 확인 + +### Requirement +- Android Studio Jellyfish 2023.3.1 + - https://developer.android.com/studio?hl=ko#get-android-studio +- 테스트용 안드로이드 폰 + - 최소 안드로이드 10 이상. + - 개발자 옵션 활성화: 설정 > 휴대전화 정보 > 소프트웨어 정보 메뉴로 진입하여 '빌드번호' 영역을 여러번 연타 + - USB 디버깅 활성화: 개발자 옵션을 활성화 한 후, 설정 > 개발자 옵션 메뉴 진입하여 'USB 디버깅' 활성화 + +### 실행확인한 환경 +- 윈도우 11 +- 안드로이드 스튜디오 2023.3.1 +- 갤럭시 S20+ (안드로이드 13 Tiramisu) + +### 사용방법 +![error](images/slide1.jpg 'slide1') +![error](images/slide2.jpg 'slide2') +![error](images/slide3.jpg 'slide3') +![error](images/slide4.jpg 'slide4') +![error](images/slide5.jpg 'slide5') +![error](images/slide6.jpg 'slide6') +![error](images/slide7.jpg 'slide7') +![error](images/slide8.jpg 'slide8') +![error](images/slide9.jpg 'slide9') +![error](images/slide10.jpg 'slide10') +![error](images/slide11.jpg 'slide11') +![error](images/slide12.jpg 'slide12') + +### 적용 기술 (현재 적용 상태): +- 객체탐지: (ONNX) YOLO nano +- 안면인식: FaceNet (모델 아키텍처 단일) +- HPE: (ONNX) YOLO-Pose nano + + +### Onnx 파일 변환 +- 원하는 pt파일을 적용하기 위해 onnx 형태로 변환이 필요하다. +- 아래 저장소 스크립트 실행하여 yolov8 pt 파일을 onnx 파일로 변환 가능하다. + - http://192.168.0.230:50003/FERMAT_TEAM/UTILITY_YOLO_EXPORT +- onnx 파일은 app/src/main/assets 폴더에 복사한다. +- Config.kt 파일의 상수 FILENAME_OD_MODEL, FILENAME_OD_LABEL, FILENAME_POSE_MODEL을 사용하고자 하는 파일명으로 수정하여 재실행한다. + +### 주요 Libraries version +- 실행에 필요한 라이브러리는 안드로이드 스튜디오에서 프로젝트 실행시 자동으로 다운로드 됨. +- com.microsoft.onnxruntime:onnxruntime-android:1.17.1 +- org.nanohttpd:nanohttpd:2.3.1 +- org.eclipse.paho:org.eclipse.paho.client.mqttv3:1.2.5 +- com.github.hannesa2:paho.mqtt.android:4.2.3 +- androidx.camera:camera-core:1.3.2 +- org.tensorflow:tensorflow-lite:2.15.0 +- com.google.mlkit:face-detection:16.1.6 diff --git a/app/.gitignore b/app/.gitignore new file mode 100644 index 0000000..42afabf --- /dev/null +++ b/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/app/build.gradle.kts b/app/build.gradle.kts new file mode 100644 index 0000000..68f68d9 --- /dev/null +++ b/app/build.gradle.kts @@ -0,0 +1,102 @@ +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.android") + kotlin("plugin.serialization") +} + +android { + namespace = "com.a2d2.mobilitygateway" + compileSdk = 34 + + defaultConfig { + applicationId = "com.a2d2.mobilitygateway" + minSdk = 29 + targetSdk = 34 + versionCode = 1 + versionName = "0.1.0" + + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles( + getDefaultProguardFile("proguard-android-optimize.txt"), + "proguard-rules.pro" + ) + } + debug { + isDebuggable = true + + //NOTE(jwkim): debug시 사용 +// resValue("string", "app_name", "AI CCTV(dev)") +// applicationIdSuffix = ".DEV" + + } + } + // 압축하지 않을 파일 + androidResources { + noCompress += "onnx" + noCompress += "tflite" + } + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + kotlinOptions { + jvmTarget = "17" + } + buildFeatures { + viewBinding = true + } +} + +dependencies { + + implementation("androidx.core:core-ktx:1.12.0") + implementation("androidx.appcompat:appcompat:1.6.1") + implementation("com.google.android.material:material:1.11.0") + implementation("androidx.constraintlayout:constraintlayout:2.1.4") + testImplementation("junit:junit:4.13.2") + androidTestImplementation("androidx.test.ext:junit:1.1.5") + androidTestImplementation("androidx.test.espresso:espresso-core:3.5.1") + + // for REST API server + implementation("org.nanohttpd:nanohttpd:2.3.1") + implementation("org.nanohttpd:nanohttpd-nanolets:2.3.1") + + // for MQTT publish/subscribe + implementation("org.eclipse.paho:org.eclipse.paho.client.mqttv3:1.2.5") + implementation("com.github.hannesa2:paho.mqtt.android:4.2.3") + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.3") + + // for Camera + val cameraxVersion = "1.3.2" + implementation("androidx.camera:camera-core:${cameraxVersion}") + implementation("androidx.camera:camera-camera2:${cameraxVersion}") + implementation("androidx.camera:camera-lifecycle:${cameraxVersion}") + implementation("androidx.camera:camera-view:${cameraxVersion}") + + // for onnx + implementation("com.microsoft.onnxruntime:onnxruntime-android:1.17.1") + + // for tensorflow lite + implementation("org.tensorflow:tensorflow-lite:2.15.0") + implementation("org.tensorflow:tensorflow-lite-gpu:2.15.0") + implementation("org.tensorflow:tensorflow-lite-gpu-api:2.15.0") + implementation("org.tensorflow:tensorflow-lite-support:0.4.4") + + // MLKit Face Detection + implementation("com.google.mlkit:face-detection:16.1.6") + + // for coroutine + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.8.0") + + // for rtsp : jitpack +// implementation("com.github.alexeyvasilyev:rtsp-client-android:2.0.11") + // FERMAT(HSJ100): LOCAL MODULE + implementation ("androidx.media3:media3-exoplayer:1.4.0") + // for rtsp : local + implementation(project(":library-client-rtsp")) +} \ No newline at end of file diff --git a/app/proguard-rules.pro b/app/proguard-rules.pro new file mode 100644 index 0000000..481bb43 --- /dev/null +++ b/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/app/src/androidTest/java/com/a2d2/mobilitygateway/ExampleInstrumentedTest.kt b/app/src/androidTest/java/com/a2d2/mobilitygateway/ExampleInstrumentedTest.kt new file mode 100644 index 0000000..7bb79cb --- /dev/null +++ b/app/src/androidTest/java/com/a2d2/mobilitygateway/ExampleInstrumentedTest.kt @@ -0,0 +1,24 @@ +package com.a2d2.mobilitygateway + +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.ext.junit.runners.AndroidJUnit4 + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + +/** + * Instrumented test, which will execute on an Android device. + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +@RunWith(AndroidJUnit4::class) +class ExampleInstrumentedTest { + @Test + fun useAppContext() { + // Context of the app under test. + val appContext = InstrumentationRegistry.getInstrumentation().targetContext + assertEquals("com.a2d2.mobilitygateway", appContext.packageName) + } +} \ No newline at end of file diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000..0fbc8fc --- /dev/null +++ b/app/src/main/AndroidManifest.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/assets/demo4.txt b/app/src/main/assets/demo4.txt new file mode 100644 index 0000000..3258e6f --- /dev/null +++ b/app/src/main/assets/demo4.txt @@ -0,0 +1,13 @@ +person +safety_helmet_off +safety_helmet_on +safety_gloves_off +safety_gloves_insulated_on_1 +safety_gloves_insulated_on_2 +safety_boots_off +safety_boots_on +safety_belt_off +safety_belt_swing_on +sign_board_information +sign_traffic_cone +signal_light_stick \ No newline at end of file diff --git a/app/src/main/assets/demo4_medium.onnx b/app/src/main/assets/demo4_medium.onnx new file mode 100644 index 0000000..57ba58c Binary files /dev/null and b/app/src/main/assets/demo4_medium.onnx differ diff --git a/app/src/main/assets/demo4_nano.onnx b/app/src/main/assets/demo4_nano.onnx new file mode 100644 index 0000000..e49be18 Binary files /dev/null and b/app/src/main/assets/demo4_nano.onnx differ diff --git a/app/src/main/assets/facenet.tflite b/app/src/main/assets/facenet.tflite new file mode 100644 index 0000000..8254aab Binary files /dev/null and b/app/src/main/assets/facenet.tflite differ diff --git a/app/src/main/assets/rtmdet.txt b/app/src/main/assets/rtmdet.txt new file mode 100644 index 0000000..73909fe --- /dev/null +++ b/app/src/main/assets/rtmdet.txt @@ -0,0 +1,24 @@ +person +truck +safety_helmet_off +safety_helmet_on +safety_gloves_off +safety_gloves_work_on +safety_gloves_insulated_on_1 +safety_gloves_insulated_on_2 +safety_gloves_insulated_on_3 +safety_gloves_insulated_on_4 +safety_rubber_insulated_sleeve_off +safety_rubber_insulated_sleeve_on +safety_boots_off +safety_boots_on +safety_belt_off +safety_belt_basic_on_1 +safety_belt_basic_on_2 +safety_belt_xband_on +safety_belt_swing_on +equipment_smartstick +sign_board_information +sign_traffic_cone +truck_lift_bucket +signal_light_stick \ No newline at end of file diff --git a/app/src/main/assets/rtmdet_tiny_32.onnx b/app/src/main/assets/rtmdet_tiny_32.onnx new file mode 100644 index 0000000..bef9c8c Binary files /dev/null and b/app/src/main/assets/rtmdet_tiny_32.onnx differ diff --git a/app/src/main/assets/yolov8.txt b/app/src/main/assets/yolov8.txt new file mode 100644 index 0000000..1f42c8e --- /dev/null +++ b/app/src/main/assets/yolov8.txt @@ -0,0 +1,80 @@ +person +bicycle +car +motorcycle +airplane +bus +train +truck +boat +traffic light +fire hydrant +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +couch +potted plant +bed +dining table +toilet +tv +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddy bear +hair drier +toothbrush \ No newline at end of file diff --git a/app/src/main/assets/yolov8l-pose.onnx b/app/src/main/assets/yolov8l-pose.onnx new file mode 100644 index 0000000..b74aae1 Binary files /dev/null and b/app/src/main/assets/yolov8l-pose.onnx differ diff --git a/app/src/main/assets/yolov8m-pose.onnx b/app/src/main/assets/yolov8m-pose.onnx new file mode 100644 index 0000000..3dc4bf7 Binary files /dev/null and b/app/src/main/assets/yolov8m-pose.onnx differ diff --git a/app/src/main/assets/yolov8m.onnx b/app/src/main/assets/yolov8m.onnx new file mode 100644 index 0000000..3826ba2 Binary files /dev/null and b/app/src/main/assets/yolov8m.onnx differ diff --git a/app/src/main/assets/yolov8n-pose.onnx b/app/src/main/assets/yolov8n-pose.onnx new file mode 100644 index 0000000..fbe0d1f Binary files /dev/null and b/app/src/main/assets/yolov8n-pose.onnx differ diff --git a/app/src/main/assets/yolov8n.onnx b/app/src/main/assets/yolov8n.onnx new file mode 100644 index 0000000..e94bf2f Binary files /dev/null and b/app/src/main/assets/yolov8n.onnx differ diff --git a/app/src/main/assets/yolov8s-pose.onnx b/app/src/main/assets/yolov8s-pose.onnx new file mode 100644 index 0000000..a9e3c76 Binary files /dev/null and b/app/src/main/assets/yolov8s-pose.onnx differ diff --git a/app/src/main/assets/yolov8s.onnx b/app/src/main/assets/yolov8s.onnx new file mode 100644 index 0000000..0543200 Binary files /dev/null and b/app/src/main/assets/yolov8s.onnx differ diff --git a/app/src/main/java/com/a2d2/mobilitygateway/BiDetect.kt b/app/src/main/java/com/a2d2/mobilitygateway/BiDetect.kt new file mode 100644 index 0000000..d2137c1 --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/BiDetect.kt @@ -0,0 +1,206 @@ +package com.a2d2.mobilitygateway + +import com.a2d2.mobilitygateway.biDetect.WatchDiscrimination +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.int +import kotlinx.serialization.json.jsonPrimitive +import org.json.JSONObject +import java.time.LocalDateTime +import java.util.Timer +import java.util.TimerTask + + +@androidx.media3.common.util.UnstableApi // FERMAT(HSJ100): LOCAL MODULE +open class BiDetect( + private val mqttManager: MQTTManager?, + private val request: JsonObject, + private val engine: EngineActivity +) { + protected val util: Util = Util() + protected open val mqttTopic = TOPIC.BI + protected open val mqttBiSensorTopic = TOPIC.SELVAS_HHS + + private var biMessage: MQTTResponseBI? = null + + private var limitTimer: Timer? = null + private var eventStop = false + protected var eventTimeout = false + private var eventComplete = false + + private var watchMessage: MQTTResponseBISensorWatchInfo? = null + + private val requestStruct = RequestJson() + private var currentStatus: String = RESPONSE_STATUS.START + + private lateinit var coroutineJob: Job + private val coroutineScope = CoroutineScope(Dispatchers.Default) + + private var workerId: String? = null + + class RequestJson { + var timerTime: Int = 0 + } + + protected fun log(level: String = "", message: String, logcat: Boolean = true) { + engine.log(level, message, logcat) + } + + protected open fun setRequestInfo(request: JsonObject) { + requestStruct.timerTime = request["timer_time"]?.jsonPrimitive?.int?: 0 + } + + open fun start() { + try { + // start 상태 전송 + publishMessage(RESPONSE_STATUS.START) + + setRequestInfo(request) + + // mqtt + mqttManager?.subscribe(topic = mqttBiSensorTopic) + + // timer + if (requestStruct.timerTime != 0) { + // timer start + val timerTask = object : TimerTask() { + override fun run() { + log("warning", "timer expired") + eventTimeout = true + } + } + limitTimer = Timer() + limitTimer?.schedule(timerTask, requestStruct.timerTime * 1000L) + } + + coroutineJob = coroutineScope.launch { + + // ready 상태 전송 + publishMessage(RESPONSE_STATUS.READY) + var currentMessage : List + + while (isActive) { + workerId = null + if (eventStop) { + publishMessage(RESPONSE_STATUS.STOP) + break + } else if (eventTimeout) { + publishMessage(RESPONSE_STATUS.TIMEOUT) + break + } else if (currentStatus == RESPONSE_STATUS.ERROR) { + break + } + currentMessage = mqttManager?.getQueneMessages()!! + + if (currentMessage.isEmpty()) { + continue + } else { + currentMessage[0]?.let { processSensorData(it) } + } + + if (biMessage != null){ + publishMessage(RESPONSE_STATUS.DETECT) + biMessage = null + } + + } + finish() + } + } + catch (e: Exception) { + publishMessage(RESPONSE_STATUS.ERROR, e.toString()) + finish() + } + } + + private suspend fun processSensorData(sensorData: String) { + withContext(Dispatchers.Default) { + try { + val parsingData = JSONObject(sensorData) + processWatch(parsingData) + + if (watchMessage != null){ + biMessage = MQTTResponseBI( + worker_id = workerId, + celvas = null, + hhs = null, + mezoo = null, + watch = watchMessage, + evaluation = null, + total_score = null + ) + } + } + catch (e: Exception) { + publishMessage(RESPONSE_STATUS.ERROR, e.toString()) + } + } + } + + private fun processWatch(sensorData: JSONObject) { + workerId = null + watchMessage = null + val discrimination = WatchDiscrimination(watchData=sensorData) + watchMessage = discrimination.getMessage() + workerId = watchMessage?.worker_id + } + + fun getIsActive(): Boolean { + return (::coroutineJob.isInitialized && coroutineJob.isActive) + } + + fun getIsComplete(): Boolean { + return (coroutineJob.isCompleted) + } + + fun getIsCancelled(): Boolean { + return (coroutineJob.isCancelled) + } + + fun stop() { + eventStop = true + } + + fun resetStop(){ + eventStop = false + } + + protected open fun finish() { + if (::coroutineJob.isInitialized) { + mqttManager?.unSubscribe(mqttBiSensorTopic) + coroutineJob.cancel() + } + eventStop = false + eventTimeout = false + eventComplete = false + + limitTimer?.cancel() + } + + protected open fun publishMessage(status: String, message: String? = null) { + log("info", javaClass.simpleName + " STATUS $status".uppercase()) + currentStatus = status + + val mqttResponse = MQTTResponseWD( + datetime = LocalDateTime.now().format(MQTT_DATETIME_FORMAT), + status = currentStatus, + message = message, + requests = request, + result = MQTTResponseWDResult( + ri_info = null, + bi = biMessage, + hpe = null, + od = null, + image_data = null, + process_time = null + ) + ) + + mqttManager?.publishMessage(mqttTopic, mqttResponse) + } +} \ No newline at end of file diff --git a/app/src/main/java/com/a2d2/mobilitygateway/Config.kt b/app/src/main/java/com/a2d2/mobilitygateway/Config.kt new file mode 100644 index 0000000..9c31a8e --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/Config.kt @@ -0,0 +1,293 @@ +package com.a2d2.mobilitygateway + +import android.Manifest +import android.graphics.Rect +import android.graphics.RectF +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put +import java.time.format.DateTimeFormatter + + +const val PERMISSION_REQUEST_CODE: Int = 101 +val REQUIRED_PERMISSIONS = arrayOf(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO) + +var OD_MODEL = ODModelFlavor.YOLOV8 + +val FILENAME_OD_MODEL: String + get() = when(OD_MODEL){ + ODModelFlavor.YOLOV8 -> "demo4_nano.onnx" + ODModelFlavor.RTMDET -> "rtmdet_tiny_32.onnx" + } + +val FILENAME_OD_LABEL: String + get() = when(OD_MODEL){ + ODModelFlavor.YOLOV8 -> "demo4.txt" + ODModelFlavor.RTMDET -> "rtmdet.txt" + } +const val FILENAME_POSE_MODEL = "yolov8n-pose.onnx" + +const val LOG_TAG_DEBUG = "A2D2_DEBUG" +const val LOG_TAG = "A2D2_ANDROID_MG" + +var use_camera = true + +const val DEFAULT_REST_PORT: Int = 51070 +const val DEFAULT_MQTT_PORT: Int = 50273 +const val DEFAULT_MQTT_BROKER_IP: String = "192.168.200.232" +const val MQTT_QOS: Int = 0 +const val MQTT_LOGIN_USERNAME: String = "admin" +const val MQTT_LOGIN_PASSWORD: String = "12341234" +val MQTT_DATETIME_FORMAT: DateTimeFormatter = DateTimeFormatter.ofPattern("yy-MM-dd HH:mm:ss.SSS") + +var USE_OD_MODEL: Boolean = true +var USE_POSE_MODEL: Boolean = true + +const val RTSP_IMAGE_WIDTH: Int = 1920 +const val RTSP_IMAGE_HEIGHT: Int = 1080 + +const val CAMERA_IMAGE_WIDTH: Int = 1920 +const val CAMERA_IMAGE_HEIGHT: Int = 1080 +const val TRAIN_IMAGE_SIZE: Int = 640 + +const val RTSP_BASE_URL = "rtsp://192.168.200.232:50299/wd" + +val RTSP_USER = null +val RTSP_PW = null + +val biOutputStatusList = arrayOf("workable", "unworkable") + +/* +위험성 탐지시 object detect 단계에서 위험을 판단하는 class list +지정된 class가 모두 detect 되면 위험으로 판단 +safety_helmet_off, safety_gloves_off + */ +val WD_WARNING_DETECT_CLASS_LIST = arrayOf("safety_helmet_off", "safety_gloves_off") +lateinit var OD_CLASSES: Array +const val VIEW_CONFIDENCE_SCORE = false + +//jwkim 임시 데이터 추가 +val biDefaultRequest = buildJsonObject{ + put("timer_time",0) +} + +// REST API를 통해 수신하는 json 구조 테스트 예제 +@Serializable +data class RestTestRequest( + val cmdName: String, + val cmdCode: Int +) + + +// OD 추론 결과 data class +data class ODResult(val classIndex: Int, val score: Float, val rectF: RectF, val rectBitmap: RectF, val bbox: ArrayList) + +// FR 추론 결과 data class +data class FRResult(var bbox: Rect, var label: String, var maskLabel: String = "") + +enum class ODModelFlavor { YOLOV8, RTMDET } + +// MQTT 응답 Json (OD) +@Serializable +data class MQTTResponse( + val datetime: String, + val status: String, + val message: String?, + val requests: JsonObject?, + val result: MQTTResponseResult? +) + +@Serializable +data class MQTTResponseResult( + val detect: Map?, + val image_data: String? +) + +@Serializable +data class MQTTResponseDetect( + val class_id: Int, + val class_name: String?, + val confidence: Float, + val bbox: ArrayList, + val image: String? +) + +// MQTT 응답 Json (FR) +@Serializable +data class MQTTResponseFr( + val datetime: String, + val status: String, + val message: String?, + val requests: JsonObject?, + val result: MQTTResponseFrResult? +) + +@Serializable +data class MQTTResponseFrResult( + val detect: Map?, + val image_data: String? +) + +@Serializable +data class MQTTResponseFrDetect( + val matched: Boolean, + val distance: Float, + val bbox: ArrayList?, + val image: String? +) + +// MQTT 응답 Json (WD) +@Serializable +data class MQTTResponseWD( + val datetime: String, + val status: String, + val message: String?, + val requests: JsonObject?, + val result: MQTTResponseWDResult? +) + +@Serializable +data class MQTTResponseWDResult( + val ri_info: MQTTResponseRIInfo?, + val bi: MQTTResponseBI?, + val hpe: MQTTResponseHpe?, + val od: MQTTResponseObject?, + val image_data: String?, + val process_time: String? +) + +@Serializable +data class MQTTResponseRIInfo( + val type: String, + val construction_type: String, + val procedure_no: Int, + val procedure_ri: Float, + val ri: Float +) + +@Serializable +data class MQTTResponseBI( + val worker_id: String?, + val celvas: Unit?, + val hhs: Unit?, + val mezoo: Unit?, + val watch: MQTTResponseBISensorWatchInfo?, + val evaluation: Unit?, + val total_score: Int? +) + +@Serializable +data class MQTTResponseBISensorWatchInfo( + val worker_id: String?, + val device_id: String?, + val heart_rate: MQTTResponseBISensorBaseInfo?, //심박수 + val boc: MQTTResponseBISensorBaseInfo?, //혈중 산소 농도 + val blood_pressure: MQTTResponseBISensorBaseInfo?, // 혈압 + val body_temperature: MQTTResponseBISensorBaseInfo?, // 체온 + val stress: MQTTResponseBISensorBaseInfo?, // 스트레스 +) + +@Serializable +data class MQTTResponseBISensorBaseInfo( + val input: String?, + val output: MQTTResponseBISensorBaseOutputInfo +) + +@Serializable +data class MQTTResponseBISensorBaseOutputInfo( + val status_idx: Int, + val status_list: Array +) + +@Serializable +data class MQTTResponseHpeDetect( + val class_id: Int, + val class_name: String, + val confidence: Float, + val bbox: ArrayList, + val image: String?, + val pose_type: Int, + val pose_level: Int +) + +@Serializable +data class MQTTResponseHpe( + val detect: ArrayList?, + val inference_time: String? +) + +@Serializable +data class MQTTResponseObject( + val detect_type: String, + val detect: ArrayList, + val inference_time: String? +) + +// MQTT TOPICs +class TOPIC { + companion object { + const val TEST = "ANDROID_TEST" + const val CON = "/AI_KEPCO/AI_OD_CON_SETUP_DETECT/REPORT" + const val FR = "/AI_KEPCO/AI_FACE_RECOGNIZE/REPORT" + const val WD = "/AI_KEPCO/AI_OD_WORK_DETECT/REPORT" + const val BI = "/AI_KEPCO/AI_BI_DETECT/REPORT" + const val DEVICE = "/AI_KEPCO/DEVICE/REPORT" + const val MEZOO = "/AI_BI_DEVICE_1" + const val SELVAS_HHS = "/PL_GW" + } +} + +// REST API URIs +class REST_URI { + companion object { + const val ROOT = "/" + } + class TEST { + companion object { + private const val PREFIX = "/test" + + const val IMAGE_PUBLISH = "$PREFIX/image_publish" + const val IMAGE_SAVE = "$PREFIX/image_save" + const val EXIT = "$PREFIX/exit" + } + } + class SERVICES { + companion object { + private const val PREFIX = "/api/services" + + const val CLASS_INFO = "$PREFIX/AE/Class-Info" + const val OD_WORK_DETECT = "$PREFIX/AE/OD-Work-Detect" + const val OD_WORK_DETECT_STOP = "$PREFIX/AE/OD-Work-Detect-Stop" + const val FR_RECOGNIZE = "$PREFIX/AE/FR-Recognize" + const val FR_RECOGNIZE_STOP = "$PREFIX/AE/FR-Recognize-Stop" + const val BI_DETECT = "$PREFIX/AE/BI-Detect" + const val BI_DETECT_STOP = "$PREFIX/AE/BI-Detect-Stop" + } + } +} + +// MQTT Response status +class RESPONSE_STATUS { + companion object { + const val START = "start" + const val READY = "ready" + const val COMPLETE = "complete" + const val NEW = "new" + const val DETECT = "detect" + const val TIMEOUT = "timeout" + const val STOP = "stop" + const val ERROR = "error" + } +} + + +class ALERT_LEVEL { + companion object { + const val NORMAL = 0 + const val WARNING = 1 + const val DANGER = 2 + } +} + + diff --git a/app/src/main/java/com/a2d2/mobilitygateway/EngineActivity.kt b/app/src/main/java/com/a2d2/mobilitygateway/EngineActivity.kt new file mode 100644 index 0000000..633ade3 --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/EngineActivity.kt @@ -0,0 +1,886 @@ +package com.a2d2.mobilitygateway + +import android.content.ContentValues +import android.graphics.Bitmap +import android.os.Build +import android.os.Bundle +import android.os.Handler +import android.os.HandlerThread +import android.os.Looper +import android.os.Message +import android.provider.MediaStore +import android.text.method.ScrollingMovementMethod +import android.util.Log +import android.util.Size +import android.view.PixelCopy +import android.view.View +import android.view.WindowInsets +import android.view.WindowInsetsController +import android.view.WindowManager +import android.widget.TextView +import android.widget.Toast +import androidx.activity.OnBackPressedCallback +import androidx.appcompat.app.AlertDialog +import androidx.appcompat.app.AppCompatActivity +import androidx.camera.core.CameraSelector +import androidx.camera.core.ImageAnalysis +import androidx.camera.core.ImageCapture +import androidx.camera.core.ImageCaptureException +import androidx.camera.core.ImageProxy +import androidx.camera.core.Preview +import androidx.camera.core.resolutionselector.AspectRatioStrategy +import androidx.camera.core.resolutionselector.ResolutionSelector +import androidx.camera.core.resolutionselector.ResolutionStrategy +import androidx.camera.lifecycle.ProcessCameraProvider +import androidx.camera.view.PreviewView +import androidx.core.content.ContextCompat +import androidx.core.view.isVisible +import com.a2d2.mobilitygateway.databinding.ActivityEngineViewBinding +import com.a2d2.mobilitygateway.faceRecognize.FrameAnalyser +import com.a2d2.mobilitygateway.objectDetect.OnnxManager +import com.a2d2.mobilitygateway.objectDetect.PoseView +import com.a2d2.mobilitygateway.objectDetect.RectView +import com.alexvas.rtsp.widget.RtspSurfaceView +import com.google.common.util.concurrent.ListenableFuture +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonObject +import java.io.FileNotFoundException +import java.time.LocalDateTime +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.random.Random +import java.util.* + + +@androidx.media3.common.util.UnstableApi // FERMAT(HSJ100): LOCAL MODULE +class EngineActivity : AppCompatActivity() { + private lateinit var binding: ActivityEngineViewBinding + + private lateinit var onnxManager: OnnxManager +// private lateinit var facenetManager: FaceNetModel + private var restManager: RESTManager? = null + private var mqttManager: MQTTManager? = null + private lateinit var util: Util + + // camerax + private lateinit var cameraExecutor: ExecutorService + private lateinit var cameraProvider: ProcessCameraProvider + private lateinit var cameraProviderFuture: ListenableFuture + private lateinit var cameraSelector: CameraSelector + private lateinit var resolutionSelector: ResolutionSelector + private lateinit var usecaseCapture: ImageCapture + private lateinit var usecasePreview: Preview + private lateinit var usecaseAnalysis: ImageAnalysis + private lateinit var usecaseFrAnalysis: ImageAnalysis + private lateinit var faceRecognizeAnalyser: FrameAnalyser + + // for UI + private lateinit var previewView: PreviewView + private lateinit var rectView: RectView + private lateinit var poseView: PoseView +// private lateinit var frRectView: FRRectView +// private lateinit var alertView: TextView + private lateinit var logView: TextView + private var imageView: Boolean = false + lateinit var rtspPreview : RtspSurfaceView + + // detect + private var isUpdateImageData: Boolean = false + var currentImageData: Bitmap? = null + get() { + return if (isUpdateImageData) { + isUpdateImageData = false + field + } else { + null + } + } + set(imageBitmap) { + field = imageBitmap + isUpdateImageData = true + } +// private lateinit var coroutineFR: FaceRecognize + private lateinit var coroutineWD: ObjectDetectWD + private lateinit var coroutineBI: BiDetect + + // FERMAT(HSJ100): RTSP_BLANK_FRAME + var rtspStatusListener = object: RtspSurfaceView.RtspStatusListener { + override fun onRtspStatusConnecting() {} + override fun onRtspStatusConnected() {} + override fun onRtspStatusDisconnecting() {} + override fun onRtspStatusDisconnected() {} + override fun onRtspStatusFailedUnauthorized() {} + override fun onRtspStatusFailed(message: String?) {} + override fun onRtspFirstFrameRendered() {} + override fun onRtspBlankFrame() { + log("warning", "Blank pipe") + } + override fun onRtspClientStarted() {} + override fun onRtspClientStopped() {} + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + binding = ActivityEngineViewBinding.inflate(layoutInflater) + setContentView(binding.root) + + // 뒤로가기 버튼 + val backPressedCallback = object : OnBackPressedCallback(true /* enabled */) { + override fun handleOnBackPressed() { + restManager?.stopServer() + servicesBIDetectStop() + servicesOdWorkDetectStop() + rtspPreview.stop() + mqttManager?.disconnect() + } + } + onBackPressedDispatcher.addCallback(this, backPressedCallback) + + // 앱 화면 항상 켜짐 + window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + + // 풀스크린 모드: 상태바, 네비게이션바 숨김 + setFullScreen() + + util = Util() + + previewView = binding.previewView + rectView = binding.rectView + poseView = binding.poseView +// alertView = binding.alertView + logView = binding.logView + logView.movementMethod = ScrollingMovementMethod() + rtspPreview = binding.svVideo + + log(message = "Init Engine.") + + val restPort = intent.getIntExtra("restPort", DEFAULT_REST_PORT) + val mqttPort = intent.getIntExtra("mqttPort", DEFAULT_MQTT_PORT) + val mqttIpAddr = intent.getStringExtra("mqttIpAddr").toString() + val mqttUser = intent.getStringExtra("mqttUser").toString() + val mqttPass = intent.getStringExtra("mqttPass").toString() + + //binding.btnMqttSimple.setOnClickListener { simplePublish() } +// binding.btnPreview.setOnClickListener { togglePreview() } + binding.btnLogView.setOnClickListener { toggleLogView() } + binding.btnImageView.setOnClickListener { toggleImage() } + + try { + startMQTT(mqttIpAddr, mqttPort, mqttUser, mqttPass) + +// facenetManager = FaceNetModel(applicationContext) +// frRectView = binding.bboxOverlay +// frRectView.setWillNotDraw( false ) +// frRectView.setZOrderOnTop( true ) +// faceRecognizeAnalyser = FrameAnalyser(frRectView, facenetManager) + + initOnnx() + + if (use_camera){ + rtspPreview.visibility = View.VISIBLE + initCamerax() + bindCameraxPreview() + bindCameraxAnalysis() + previewView.visibility = View.INVISIBLE + } + else{ + previewView.visibility = View.VISIBLE + } + + rectView.visibility = View.INVISIBLE + poseView.visibility = View.INVISIBLE + +// capture bind시 갤러시S23울트라(안드로이드14)에서 카메라 동작안하는 문제 있음 +// capture는 카메라 테스트를 위한 기능이며 실제 MG 동작에선 불필요 +// bindCameraxCapture() + + startREST(restPort) + + Timer().schedule(object : TimerTask() { + override fun run() { + servicesBIDetect(biDefaultRequest) + } + }, 1000) + + log(message = "Start Engine.") + + }catch (e: Exception){ + // alert + val builder: AlertDialog.Builder = AlertDialog.Builder(this) + + builder.setTitle("error").setMessage(e.message) + builder.setPositiveButton("확인") { dialog, id -> + dialog.dismiss() + finish() + } + builder.create() + builder.show() + } + } + + private fun toggleLogView() { + if (logView.isVisible) { + logView.visibility = View.INVISIBLE + binding.btnLogView.text = getString(R.string.logview_on) + } + else { + logView.visibility = View.VISIBLE + binding.btnLogView.text = getString(R.string.logview_off) + } + } + + private fun toggleImage() { + if (use_camera){ + if (previewView.isVisible) { + rtspPreview.visibility = View.INVISIBLE + previewView.visibility = View.INVISIBLE + poseView.visibility = View.INVISIBLE + rectView.visibility = View.INVISIBLE +// toggleFrView() + binding.btnImageView.text = getString(R.string.image_on) + } + else{ + rtspPreview.visibility = View.VISIBLE + previewView.visibility = View.VISIBLE + poseView.visibility = View.VISIBLE + rectView.visibility = View.VISIBLE +// toggleFrView() + binding.btnImageView.text = getString(R.string.image_off) + } + } + else { + if (rectView.isVisible) { + imageView = false +// alertView.visibility = View.INVISIBLE + previewView.visibility = View.VISIBLE + togglePoseView() + toggleRectView() +// toggleFrView() +// toggleCamView() + binding.btnImageView.text = getString(R.string.image_on) + } else { + imageView = true + previewView.visibility = View.INVISIBLE + togglePoseView() + toggleRectView() +// toggleFrView() +// toggleCamView() + binding.btnImageView.text = getString(R.string.image_off) + } + } + } + + private fun togglePoseView() { + if (poseView.isVisible) { + poseView.visibility = View.INVISIBLE + } + else { + poseView.visibility = View.VISIBLE + } + } + + private fun toggleRectView() { + if (rectView.isVisible) { + rectView.visibility = View.INVISIBLE + } + else { + rectView.visibility = View.VISIBLE + } + } + +// private fun toggleFrView(){ +// if (frRectView.isVisible) { +// frRectView.visibility = View.INVISIBLE +// } +// else { +// frRectView.visibility = View.VISIBLE +// } +// } + +// fun setAlertView(level: Int) { +// if (imageView){ +// when(level) { +// ALERT_LEVEL.NORMAL -> { +// runOnUiThread { +// alertView.visibility = View.INVISIBLE +// } +// } +// ALERT_LEVEL.WARNING -> { +// runOnUiThread { +// alertView.visibility = View.VISIBLE +// alertView.text = getString(R.string.alert_warning) +// alertView.setBackgroundColor(Color.parseColor("#88FFFF00")) +// } +// } +// ALERT_LEVEL.DANGER -> { +// runOnUiThread { +// alertView.visibility = View.VISIBLE +// alertView.text = getString(R.string.alert_danger) +// alertView.setBackgroundColor(Color.parseColor("#88FF0000")) +// } +// } +// else -> {} +// } +// } +// } + + fun log(level: String = "", message: String, logcat: Boolean = true) { + //logView.post { + runOnUiThread { + logView.append("\n${if(level.isNotEmpty()) ">> $level | " else ""}$message") + } + + if (logcat) { + when (level.lowercase()) { + "info" -> Log.i(LOG_TAG, message) + "warning" -> Log.w(LOG_TAG, message) + "error" -> Log.e(LOG_TAG, message) + else -> Log.d(LOG_TAG_DEBUG, message) + } + } + } + + // camerax image capture 하여 폰에 저장하는 테스트 함수 + private fun imageSave(filename: String) { + val contentValues = ContentValues().apply { + put(MediaStore.MediaColumns.DISPLAY_NAME, filename) + put(MediaStore.MediaColumns.MIME_TYPE, "image/jpeg") + // /storage/self/primary/Pictures/CameraX-Image + put(MediaStore.Images.Media.RELATIVE_PATH, "Pictures/CameraX-Image") + } + val outputOptions = ImageCapture.OutputFileOptions + .Builder(contentResolver, + MediaStore.Images.Media.EXTERNAL_CONTENT_URI, + contentValues) + .build() + + // takePicture는 OnImageSaved와 OnImageCaptured 등 2가지 옵션이 있음. + usecaseCapture.takePicture(outputOptions, cameraExecutor, + object : ImageCapture.OnImageSavedCallback { + override fun onImageSaved(outputFileResults: ImageCapture.OutputFileResults) { + log("info", "image file saved. (uri: ${outputFileResults.savedUri})") + } + + override fun onError(exception: ImageCaptureException) { + log("error", "IMAGE FILE SAVE FAIL:: $exception") + } + }) + } + + private val pubImageTest = object : ImageCapture.OnImageCapturedCallback() { + override fun onCaptureSuccess(image: ImageProxy) { + log("info", "camera image captured") + +// val map = HashMap() +// val encodedImage = encodeToBase64(image) +// +// map["datetime"] = LocalDateTime.now().format(MQTT_DATETIME_FORMAT) +// map["status"] = RESPONSE_STATUS.DETECT +// map["result"] = mapOf("image_data" to encodedImage) +// +// mqttManager.publishMessage(TOPIC.WD, map) + + val mqttResponse = MQTTResponse( + datetime = LocalDateTime.now().format(MQTT_DATETIME_FORMAT), + status = RESPONSE_STATUS.DETECT, + message = null, + requests = null, + result = MQTTResponseResult( + detect = null, + image_data = util.encodeToBase64(image, forceScale = true) + ) + ) + + mqttManager?.publishMessage(TOPIC.WD, mqttResponse) + + image.close() + super.onCaptureSuccess(image) + } + + override fun onError(exception: ImageCaptureException) { + log("error", "CAMERA IMAGE CAPTURE FAIL:: $exception") + + super.onError(exception) + } + } + + private val pubRandomString = fun(image: ImageProxy) { + log("info", "camera image captured") + + val map = HashMap() + + map["datetime"] = LocalDateTime.now().format(MQTT_DATETIME_FORMAT) + map["status"] = "detect" + map["randomString"] = util.getRandomString(Random.nextInt(1000, 8000)) + + mqttManager?.publishMessage(TOPIC.TEST, map) + + image.close() + } + + // camerax 이미지 capture 후 callback 실행 + private fun imageCapture(callback: ImageCapture.OnImageCapturedCallback? = null) { + usecaseCapture.takePicture(ContextCompat.getMainExecutor(this), + callback?: object : ImageCapture.OnImageCapturedCallback() { + override fun onCaptureSuccess(image: ImageProxy) = + pubRandomString(image).also { + super.onCaptureSuccess(image) + } + + override fun onError(exception: ImageCaptureException) { + log("error", "CAMERA IMAGE CAPTURE FAIL:: $exception") + + super.onError(exception) + } + }) + } + + private fun initCamerax() { + try { + cameraProviderFuture = ProcessCameraProvider.getInstance(this) + cameraProvider = cameraProviderFuture.get() + cameraExecutor = Executors.newSingleThreadExecutor() + + cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA + resolutionSelector = ResolutionSelector.Builder() + // resolutionStrategy와 aspetcRatioStrategy를 함께 설정해야 지정한 해상도가 제대로 반영된다. + .setResolutionStrategy(ResolutionStrategy( + Size(CAMERA_IMAGE_WIDTH, CAMERA_IMAGE_HEIGHT), + ResolutionStrategy.FALLBACK_RULE_CLOSEST_HIGHER_THEN_LOWER)) + .setAspectRatioStrategy(AspectRatioStrategy.RATIO_16_9_FALLBACK_AUTO_STRATEGY).build() + + usecasePreview = Preview.Builder().setResolutionSelector(resolutionSelector).build() + usecasePreview.setSurfaceProvider(previewView.surfaceProvider) + + usecaseCapture = ImageCapture.Builder().setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY) + .setResolutionSelector(resolutionSelector).build() + + usecaseAnalysis = ImageAnalysis.Builder().setResolutionSelector(resolutionSelector) + .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST).build().also { + it.setAnalyzer(cameraExecutor) { imageProxy -> + currentImageData = imageProxy.toBitmap() + imageProxy.close() + } + } + +// usecaseFrAnalysis = ImageAnalysis.Builder().setResolutionSelector(resolutionSelector) +// .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) +// .setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888).build().also { +// it.setAnalyzer(cameraExecutor, faceRecognizeAnalyser) +// } + + cameraProvider.unbindAll() + } + catch (e: Exception) { + log("error", "START CAMERA FAIL:: $e") + throw Throwable("can't start camera") + } + } + + fun bindCameraxPreview() { + if (cameraProvider.isBound(usecasePreview)) { + log("warning", "camerax-preview is bound already") + return + } + + try { + cameraProviderFuture.addListener({ + cameraProvider.bindToLifecycle(this, cameraSelector, usecasePreview) + }, ContextCompat.getMainExecutor(this)) + } + catch (e: Exception) { + log("error", "PREVIEW USECASE BINDING FAIL:: $e") + throw Throwable("can't bind camerax-preview") + } + } + + fun unbindCameraxPreview() { + if (!cameraProvider.isBound(usecasePreview)) { + log("warning", "camerax-preview is unbound already") + return + } + + try { + cameraProviderFuture.addListener({ + cameraProvider.unbind(usecasePreview) + }, ContextCompat.getMainExecutor(this)) + } + catch (e: Exception) { + log("error", "PREVIEW USECASE UNBINDING FAIL:: $e") + throw Throwable("can't unbind camerax-preview") + } + } + + fun bindCameraxCapture() { + if (cameraProvider.isBound(usecaseCapture)) { + log("warning", "camerax-capture is bound already") + return + } + + try { + cameraProviderFuture.addListener({ + cameraProvider.bindToLifecycle(this, cameraSelector, usecaseCapture) + }, ContextCompat.getMainExecutor(this)) + } + catch (e: Exception) { + log("error", "IMAGE CAPTURE USECASE BINDING FAIL:: $e") + throw Throwable("can't bind camerax-capture") + } + } + + fun bindCameraxAnalysis() { + if (cameraProvider.isBound(usecaseAnalysis)) { + log("warning", "camerax-analysis is bound already") + return + } + + try { + isUpdateImageData = false + + cameraProviderFuture.addListener({ + cameraProvider.bindToLifecycle(this, cameraSelector, usecaseAnalysis) + }, ContextCompat.getMainExecutor(this)) + } + catch (e: Exception) { + log("error", "IMAGE ANALYSIS USECASE BINDING FAIL:: $e") + throw Throwable("can't bind camerax-analysis") + } + } + + fun unbindCameraxAnalysis() { + if (!cameraProvider.isBound(usecaseAnalysis)) { + log("warning", "camerax-analysis is unbound already") + return + } + + try { + cameraProviderFuture.addListener({ + cameraProvider.unbind(usecaseAnalysis) + }, ContextCompat.getMainExecutor(this)) + } + catch (e: Exception) { + log("error", "IMAGE ANALYSIS USECASE UNBINDING FAIL:: $e") + throw Throwable("can't unbind camerax-analysis") + } + } + + fun bindCameraxFrAnalysis() { + if (cameraProvider.isBound(usecaseFrAnalysis)) { + log("warning", "camerax-faceRecognize-analysis is bound already") + return + } + + try { + cameraProviderFuture.addListener({ + cameraProvider.bindToLifecycle(this, cameraSelector, usecaseFrAnalysis) + }, ContextCompat.getMainExecutor(this)) + } + catch (e: Exception) { + log("error", "IMAGE FR-ANALYSIS USECASE BINDING FAIL:: $e") + throw Throwable("can't bind camerax-faceRecognize-analysis") + } + } + + fun unbindCameraxFrAnalysis() { + if (!cameraProvider.isBound(usecaseFrAnalysis)) { + log("warning", "camerax-faceRecognize-analysis is unbound already") + return + } + + try { + cameraProviderFuture.addListener({ + cameraProvider.unbind(usecaseFrAnalysis) + }, ContextCompat.getMainExecutor(this)) + } + catch (e: Exception) { + log("error", "IMAGE FR ANALYSIS USECASE UNBINDING FAIL:: $e") + throw Throwable("can't unbind camerax-faceRecognize-analysis") + } + } + + private fun initOnnx() { + try { + onnxManager = OnnxManager(this, OD_MODEL) + + rectView.setClassLabel(onnxManager.classes) + OD_CLASSES = onnxManager.classes + + if (USE_OD_MODEL){log("info","$FILENAME_OD_MODEL loaded")} + + if (USE_POSE_MODEL){log("info","$FILENAME_POSE_MODEL loaded")} + + } + catch (e: FileNotFoundException) { + Log.e(LOG_TAG, "FILE NOT FOUND:: ${e.message}") + Toast.makeText(this, "FILE NOT FOUND:: ${e.message}", Toast.LENGTH_LONG).show() + finish() + } + } + + private fun startMQTT(ip: String, port: Int, username: String, password: String) { + mqttManager = MQTTManager(this, ip, port) + mqttManager?.connect(username, password) + } + + private fun startREST(port: Int) { + //TODO(JWKIM) : rest 객체 생성, 시작 따로 작업해야함 + restManager = RESTManager.getServer(this, port) + + restManager?.addRoute(REST_URI.ROOT, TitleRestHandler::class.java) + +// restManager.addRoute(REST_URI.TEST.IMAGE_PUBLISH, BaseRestHandler::class.java, runTestCommand) +// restManager.addRoute(REST_URI.TEST.IMAGE_SAVE, BaseRestHandler::class.java, runTestCommand) +// restManager.addRoute(REST_URI.TEST.EXIT, BaseRestHandler::class.java, runTestCommand) +// +// restManager.addRoute(REST_URI.SERVICES.CLASS_INFO, ClassInfoRestHandler::class.java, null, this) +// restManager.addRoute(REST_URI.SERVICES.FR_RECOGNIZE, FrRestHandler::class.java, runServices) +// restManager.addRoute(REST_URI.SERVICES.FR_RECOGNIZE_STOP, FrRestHandler::class.java, runServices) + restManager?.addRoute(REST_URI.SERVICES.OD_WORK_DETECT, WDRestHandler::class.java, runServices) + restManager?.addRoute(REST_URI.SERVICES.OD_WORK_DETECT_STOP, WDRestHandler::class.java, runServices) + restManager?.addRoute(REST_URI.SERVICES.BI_DETECT, BIRestHandler::class.java, runServices) + restManager?.addRoute(REST_URI.SERVICES.BI_DETECT_STOP, BIRestHandler::class.java, runServices) + + restManager?.startServer() + } + + // REST API를 통한 테스트를 위한 handler + private val runTestCommand = object : Handler(Looper.getMainLooper()) { + override fun handleMessage(msg: Message) { + super.handleMessage(msg) + + val uri = (msg.obj as List)[0] as String + + when (uri) { + REST_URI.TEST.IMAGE_SAVE -> { + val timeString = LocalDateTime.now().toString() + imageSave("ANDROID_MG_$timeString") + } + REST_URI.TEST.IMAGE_PUBLISH -> imageCapture(pubImageTest) + REST_URI.TEST.EXIT -> finishAffinity() + } + } + } + + private val runServices = object : Handler(Looper.getMainLooper()) { + override fun handleMessage(msg: Message) { + super.handleMessage(msg) + + val uri = (msg.obj as List)[0] as String + val body = (msg.obj as List)[1] as String + val json = Json.parseToJsonElement(body).jsonObject + + log("debug", "ENTER looper handler for uri '$uri'") + + when (uri) { +// REST_URI.SERVICES.CLASS_INFO -> {} +// REST_URI.SERVICES.FR_RECOGNIZE -> servicesFrRecognize(json) +// REST_URI.SERVICES.FR_RECOGNIZE_STOP -> servicesFrRecognizeStop() + REST_URI.SERVICES.OD_WORK_DETECT -> servicesOdWorkDetect(json) + REST_URI.SERVICES.OD_WORK_DETECT_STOP -> servicesOdWorkDetectStop() + REST_URI.SERVICES.BI_DETECT -> servicesBIDetect(json) + REST_URI.SERVICES.BI_DETECT_STOP -> servicesBIDetectStop() + else -> {7 + notImplemented() + } + } + } + } + + private fun servicesOdWorkDetect(request: JsonObject) { + if(::coroutineWD.isInitialized && coroutineWD.getIsActive()) { + log("warning", "OD-WORK-DETECT is already working") + coroutineWD.stop() +// coroutineWD.terminate() // FERMAT(JWKIM):stop message 안나옴! + while(!coroutineWD.getIsComplete()) { + Thread.sleep(1000) + } + } + coroutineWD = ObjectDetectWD(onnxManager, mqttManager, request, this) + coroutineWD.start() + + } + + private fun servicesOdWorkDetectStop() { + if(!::coroutineWD.isInitialized) { + log("warning", "coroutineWD is not initialized") + return + } + + if(!coroutineWD.getIsActive()) { + log("warning", "OD-WORK-DETECT is not working") + return + } + + coroutineWD.stop() + + while(coroutineWD.getIsActive()){ + Thread.sleep(100) + } + } + + private fun servicesBIDetect(request: JsonObject) { + if(::coroutineBI.isInitialized && coroutineBI.getIsActive()) { + log("warning", "BI-DETECT is already working") + coroutineBI.stop() + while(!coroutineBI.getIsComplete()) { + Thread.sleep(1000) + } + } + + coroutineBI = BiDetect(mqttManager, request, this) + coroutineBI.start() + } + + private fun servicesBIDetectStop() { + if(!::coroutineBI.isInitialized) { + log("warning", "coroutineBI is not initialized") + return + } + + if(!coroutineBI.getIsActive()) { + log("warning", "coroutineBI is not working") + return + } + + coroutineBI.stop() + + while(coroutineBI.getIsActive()){ + Thread.sleep(100) + } + } + +// private fun servicesFrRecognize(request: JsonObject) { +// if(::coroutineFR.isInitialized && coroutineFR.getIsActive()) { +// log("warning", "FR-Recognize is already working") +// return +// } +// +// coroutineFR = FaceRecognize(facenetManager, mqttManager, request, this) +// coroutineFR.startFrRecognize() +// } +// +// private fun servicesFrRecognizeStop() { +// if(!::coroutineFR.isInitialized) { +// log("warning", "coroutineFR is not initialized") +// return +// } +// +// if(!coroutineFR.getIsActive()) { +// log("warning", "FR-Recognize is not working") +// return +// } +// +// coroutineFR.stopFrRecognize() +// } + + private fun notImplemented() { + Log.d(LOG_TAG_DEBUG, "ENTER ${Throwable().stackTrace[0].methodName}()") + mqttManager!!?.publishError(TOPIC.FR, "Not Implemented") + } + + fun updateRectView(results: ArrayList? = null) { + if (rectView.isVisible) { + rectView.transformRect(results ?: arrayListOf()) + rectView.invalidate() + } + } + + fun updatePoseView(results: ArrayList? = null) { + if (poseView.isVisible) { + poseView.setList(results ?: arrayListOf()) + poseView.invalidate() + } + } + +// fun updateFrRectView(results: ArrayList? = null) { +// frRectView.faceBoundingBoxes = results +// frRectView.invalidate() +// } + + // invalidate시 view를 refresh 할 수 있음. + fun updatePreviewView() { + previewView.invalidate() + } + + fun getModel() = onnxManager + + private fun setFullScreen() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + window.setDecorFitsSystemWindows(false) + window.insetsController?.hide(WindowInsets.Type.systemBars() or WindowInsets.Type.navigationBars()) + window.insetsController?.systemBarsBehavior = WindowInsetsController.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE + } + else { + window.decorView.systemUiVisibility = + (View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY or + View.SYSTEM_UI_FLAG_FULLSCREEN or + View.SYSTEM_UI_FLAG_HIDE_NAVIGATION or + View.SYSTEM_UI_FLAG_LAYOUT_STABLE or + View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN or + View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION) + } + } + + override fun onWindowFocusChanged(hasFocus: Boolean) { + if(!hasFocus) return + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + window.setDecorFitsSystemWindows(false) + window.insetsController?.hide(WindowInsets.Type.systemBars() or WindowInsets.Type.navigationBars()) + window.insetsController?.systemBarsBehavior = WindowInsetsController.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE + } + else { + window.decorView.systemUiVisibility = + (View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY or + View.SYSTEM_UI_FLAG_FULLSCREEN or + View.SYSTEM_UI_FLAG_HIDE_NAVIGATION or + View.SYSTEM_UI_FLAG_LAYOUT_STABLE or + View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN or + View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION) + } + } + + public override fun onDestroy() { + rtspPreview.stop() + servicesBIDetectStop() + servicesOdWorkDetectStop() + if (mqttManager?.checkConnection() == true){ + mqttManager?.disconnect() + } + restManager?.stopServer() + +// cameraProvider.unbindAll() +// cameraExecutor.shutdown() + + Log.d(LOG_TAG_DEBUG, "----- END ${javaClass.simpleName} -----") + + super.onDestroy() + } + + fun getSnapshot(): Bitmap? { +// if (DEBUG) Log.v(TAG, "getSnapshot()") + val surfaceBitmap = Bitmap.createBitmap(1920, 1080, Bitmap.Config.ARGB_8888) + val lock = Object() + + val success = AtomicBoolean(false) + val thread = HandlerThread("PixelCopyHelper") + thread.start() + val sHandler = Handler(thread.looper) + val listener = PixelCopy.OnPixelCopyFinishedListener { copyResult -> + success.set(copyResult == PixelCopy.SUCCESS) + synchronized (lock) { + lock.notify() + } + } + + synchronized (lock) { + PixelCopy.request(binding.svVideo.holder.surface, surfaceBitmap, listener, sHandler) + lock.wait() + } + thread.quitSafely() + return if (success.get()) surfaceBitmap else null + } +} diff --git a/app/src/main/java/com/a2d2/mobilitygateway/FaceRecognize.kt b/app/src/main/java/com/a2d2/mobilitygateway/FaceRecognize.kt new file mode 100644 index 0000000..1278c36 --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/FaceRecognize.kt @@ -0,0 +1,360 @@ +package com.a2d2.mobilitygateway + +import android.graphics.Bitmap +import android.graphics.Rect +import android.util.Log +import com.a2d2.mobilitygateway.faceRecognize.FaceNetModel +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.face.Face +import com.google.mlkit.vision.face.FaceDetection +import com.google.mlkit.vision.face.FaceDetectorOptions +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.boolean +import kotlinx.serialization.json.int +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import java.time.LocalDateTime +import java.util.Timer +import java.util.TimerTask +import kotlin.math.pow +import kotlin.math.sqrt + + +@androidx.media3.common.util.UnstableApi // FERMAT(HSJ100): LOCAL MODULE +class FaceRecognize( + private val facenetManager: FaceNetModel, + private val mqttManager: MQTTManager, + private val request: JsonObject, + private val engine: EngineActivity +) { + private val util: Util = Util() + private val mqttTopic = TOPIC.FR + + private var limitTimer: Timer? = null + private var eventStop = false + private var eventTimeout = false + private var eventComplete = false + + private val requestStruct = RequestJson() + private val detectedInfo = LinkedHashMap() // detect 해야할 class 정보 + private var currentStatus: String = RESPONSE_STATUS.START + private var currentProcessIndex: Int = 0 // 현재 detect 중인 worker index + + private lateinit var coroutineJob: Job + private val coroutineScope = CoroutineScope(Dispatchers.Default) + + private val realTimeOpts = FaceDetectorOptions.Builder() + .setPerformanceMode( FaceDetectorOptions.PERFORMANCE_MODE_FAST ) + .build() + private val detector = FaceDetection.getClient(realTimeOpts) + + companion object { + const val DETECTED_ID = 0 + const val DETECTED_WORKER_IMAGE = 1 + const val DETECTED_TARGET_IMAGE = 2 + } + + class RequestJson { + var limit_time_min: Int = 0 + var report_unit: Boolean = false + var crop_image: Boolean = false + var snap_shot: Boolean = false + var detect_list = LinkedHashMap>() // index, + } + + private fun log(level: String = "", message: String, logcat: Boolean = true) { + engine.log(level, message, logcat) + } + + private fun setRequestInfo(request: JsonObject) { + val targets = request["targets"]?.jsonObject + + requestStruct.limit_time_min = request["limit_time_min"]?.jsonPrimitive?.int?: 0 + requestStruct.report_unit = request["report_unit"]?.jsonPrimitive?.boolean?: false + requestStruct.crop_image = targets?.get("crop_image")?.jsonPrimitive?.boolean?: false + requestStruct.snap_shot = targets?.get("snap_shot")?.jsonPrimitive?.boolean?: false + + var idx = 0 + targets?.get("detect_list")!!.jsonArray.forEach { + val id = it.jsonObject["id"]?.jsonPrimitive?.content?: "" + val worker_image = it.jsonObject["worker_image"]?.jsonPrimitive?.content?: "" + val target_image = it.jsonObject["target_image"]?.jsonPrimitive?.content?: "" + + requestStruct.detect_list.put(idx++, arrayListOf(id, worker_image, target_image)) + detectedInfo.put(id, null) + } + } + + fun startFrRecognize() { + try { + publishMessage(RESPONSE_STATUS.START) + + setRequestInfo(request) + + // detect할 작업자가 없으면 종료 + if (requestStruct.detect_list.size == 0) { + log("warning", "there is no worker to detect") + publishMessage(RESPONSE_STATUS.ERROR, "there is no worker to detect") + return + } + + // timer start + val timerTask = object : TimerTask() { + override fun run() { + log("warning", "FrRecognize timer expired") + eventTimeout = true + } + } + limitTimer = Timer() + limitTimer?.schedule(timerTask, requestStruct.limit_time_min * 60 * 1000L) + + //engine.bindCameraxFrAnalysis() + + currentProcessIndex = 0 + + coroutineJob = coroutineScope.launch { + + // ready 상태 전송 + publishMessage(RESPONSE_STATUS.READY) + + processWorkerImage() + + while (isActive) { + if (eventStop) { + publishMessage(RESPONSE_STATUS.STOP) + break + } else if (eventTimeout) { + publishMessage(RESPONSE_STATUS.TIMEOUT) + break + } else if (eventComplete) { + val snapshot = if (requestStruct.snap_shot) + // 받은거 그대로 실어보냄 + requestStruct.detect_list[currentProcessIndex]?.get(DETECTED_TARGET_IMAGE) + else + null + publishMessage(RESPONSE_STATUS.COMPLETE, snapshot = snapshot) + break + } else if (currentStatus == RESPONSE_STATUS.ERROR) { + break + } + } + finishFrRecognize() + } + } + catch (e: Exception) { + publishMessage(RESPONSE_STATUS.ERROR, e.toString()) + finishFrRecognize() + } + } + + fun getIsActive(): Boolean { + return (::coroutineJob.isInitialized && coroutineJob.isActive) + } + + fun stopFrRecognize() { + eventStop = true + } + + private fun finishFrRecognize() { + coroutineJob.cancel() + + eventStop = false + eventTimeout = false + eventComplete = false + + limitTimer?.cancel() + + //engine.unbindCameraxFrAnalysis() + } + + // worker_image에서 face detection + private fun processWorkerImage() { + val id = requestStruct.detect_list[currentProcessIndex]?.get(DETECTED_ID)!! + val worker_image = requestStruct.detect_list[currentProcessIndex]?.get(DETECTED_WORKER_IMAGE)!! + val workerBitmap = util.decodeFromBase64(worker_image) + + detector.process(InputImage.fromBitmap(workerBitmap, 0)) + .addOnSuccessListener { faces -> + if (faces.size > 0) { + coroutineScope.launch { + processTargetImage(getEmbedding(workerBitmap, faces.first().boundingBox)) + } + } + else { + publishMessage(RESPONSE_STATUS.ERROR, "couldn't detect any face from worker_image of '${id}'") + } + } + } + + // target_image에서 face detection + private fun processTargetImage(workerEmbedding: FloatArray) { + val id = requestStruct.detect_list[currentProcessIndex]?.get(DETECTED_ID)!! + val target_image = requestStruct.detect_list[currentProcessIndex]?.get(DETECTED_TARGET_IMAGE)!! + val targetBitmap = util.decodeFromBase64(target_image) + + detector.process(InputImage.fromBitmap(targetBitmap, 0)) + .addOnSuccessListener { faces -> + if (faces.size > 0) { + Log.d(LOG_TAG_DEBUG, "detected ${faces.size} ${if(faces.size>1) "faces" else "face"} from target_image of '$id'") + coroutineScope.launch { + recognizeFace(faces, targetBitmap, workerEmbedding) + } + } + else { + publishMessage(RESPONSE_STATUS.ERROR, "couldn't detect any face from target_image of '${id}'") + } + } + } + + // Use any one of the two metrics, "cosine" or "l2" + private val metricToBeUsed = "cosine" + private suspend fun recognizeFace(faces: List, target_image: Bitmap, workerEmbedding: FloatArray) { + /** + * :param faces: target_image에서 검출된 faces + * :param workerEmbedding: worker_image에서 검출된 face의 embedding data + */ + + val id = requestStruct.detect_list[currentProcessIndex]?.get(DETECTED_ID)!! + + withContext(Dispatchers.Default) { + try { + var subject: FloatArray + var score: Float + var selectedScore: Float? = null + lateinit var selectedFace: Face + + // target_image에서 검출된 face 중에서 가장 score가 높은 face를 선택 + for (face in faces) { + subject = getEmbedding(target_image, face.boundingBox) + + // Compute the L2 norm and then append it to the ArrayList. + score = if (metricToBeUsed == "cosine") { + cosineSimilarity(subject, workerEmbedding) + } else { + L2Norm(subject, workerEmbedding) + } + + // Calculate the minimum L2 distance from the stored average L2 norms. + if (metricToBeUsed == "cosine") { + // In case of cosine similarity, choose the highest value. + if (score > facenetManager.model.cosineThreshold) { + if (selectedScore == null || score > selectedScore) { + selectedScore = score + selectedFace = face + } + } else { + Log.d(LOG_TAG_DEBUG, "different! ($id:: $score)") + } + } else { + // In case of L2 norm, choose the lowest value. + if (score < facenetManager.model.l2Threshold) { + if (selectedScore == null || score < selectedScore) { + selectedScore = score + selectedFace = face + } + } else { + Log.d(LOG_TAG_DEBUG, "different! ($id:: $score)") + } + } + } + + // 동일 얼굴을 발견한 경우 + if (selectedScore != null) { + log("info", "recognized face for worker '$id', score: $selectedScore") + + val cropBitmap = util.cropBitmapUsingRect(target_image, selectedFace.boundingBox) + val cropImage = if (requestStruct.crop_image) util.encodeToBase64(cropBitmap) else null + + detectedInfo.set(id, MQTTResponseFrDetect( + matched = true, + distance = selectedScore, + bbox = arrayListOf( + selectedFace.boundingBox.left, selectedFace.boundingBox.top, + selectedFace.boundingBox.right, selectedFace.boundingBox.bottom + ), + image = cropImage + )) + + if (requestStruct.report_unit) { + publishMessage(RESPONSE_STATUS.NEW) + currentStatus = RESPONSE_STATUS.READY + } + } + else { + // 동일 얼굴을 발견하지 못한 경우 + detectedInfo.set(id, MQTTResponseFrDetect( + matched = false, + distance = 0.0f, + bbox = null, + image = null + )) + + publishMessage(RESPONSE_STATUS.ERROR, "can't find worker face for '$id'") + } + + // 모든 요소를 detect 했다면 + if (!detectedInfo.values.contains(null)) { + eventComplete = true + } + else { + currentProcessIndex++ + if (currentProcessIndex < requestStruct.detect_list.size) { + // 다음 작업자 + processWorkerImage() + } + else { + // 이 경우는 발생하면 안됨 + publishMessage(RESPONSE_STATUS.ERROR, "index error at $id") + } + } + } catch (e: Exception) { + log("error", "Exception in recognizeFace(): ${e.message}") + publishMessage(RESPONSE_STATUS.ERROR, "Exception in recognizeFace(): ${e.message}") + } + } + } + + // Compute the L2 norm of ( x2 - x1 ) + private fun L2Norm(x1: FloatArray, x2: FloatArray): Float { + return sqrt( x1.mapIndexed{ i , xi -> (xi - x2[ i ]).pow( 2 ) }.sum() ) + } + + // Compute the cosine of the angle between x1 and x2. + private fun cosineSimilarity(x1: FloatArray, x2: FloatArray): Float { + val mag1 = sqrt( x1.map { it * it }.sum() ) + val mag2 = sqrt( x2.map { it * it }.sum() ) + val dot = x1.mapIndexed{ i , xi -> xi * x2[ i ] }.sum() + + return dot / (mag1 * mag2) + } + + private fun getEmbedding(image: Bitmap, bbox : Rect) : FloatArray { + return facenetManager.getFaceEmbedding(util.cropBitmapUsingRect(image, bbox)) + } + + + private fun publishMessage(status: String, message: String? = null, snapshot: String? = null) { + log("info", javaClass.simpleName + " STATUS $status".uppercase()) + currentStatus = status + + val mqttResponse = MQTTResponseFr( + datetime = LocalDateTime.now().format(MQTT_DATETIME_FORMAT), + status = currentStatus, + message = message, + requests = request, + result = MQTTResponseFrResult( + detect = detectedInfo, + image_data = snapshot + ) + ) + + mqttManager.publishMessage(mqttTopic, mqttResponse) + } +} \ No newline at end of file diff --git a/app/src/main/java/com/a2d2/mobilitygateway/HpeClassification.kt b/app/src/main/java/com/a2d2/mobilitygateway/HpeClassification.kt new file mode 100644 index 0000000..447ca6e --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/HpeClassification.kt @@ -0,0 +1,294 @@ +package com.a2d2.mobilitygateway + +import android.graphics.PointF +import android.util.Log +import kotlin.math.* + + +enum class KeyPoint(val idx: Int, val x: Int, val y: Int, val c: Int) { + NOSE(0, 5, 6, 7), + RIGHT_EYE(1, 8, 9, 10), + LEFT_EYE(2, 11, 12, 13), + RIGHT_EAR(3, 14, 15, 16), + LEFT_EAR(4, 17, 18, 19), + RIGHT_SHOULDER(5, 20, 21, 22), + LEFT_SHOULDER(6, 23, 24, 25), + RIGHT_ELBOW(7, 26, 27, 28), + LEFT_ELBOW(8, 29, 30, 31), + RIGHT_WRIST(9, 32, 33, 34), + LEFT_WRIST(10, 35, 36, 37), + RIGHT_HIP(11, 38, 39, 40), + LEFT_HIP(12, 41, 42, 43), + RIGHT_KNEE(13, 44, 45, 46), + LEFT_KNEE(14, 47, 48, 49), + RIGHT_FEET(15, 50, 51, 52), + LEFT_FEET(16, 53, 54, 55); +} + + +class HPELevel { + companion object { + const val NORMAL = 0 + const val FALLDOWN = 8 + const val CROSSARM = 9 + } +} + +class HPETypeMask { + companion object { + const val NORMAL = 0x0000 + const val FALLDOWN = 0x0080 + const val CROSSARM = 0x0100 + } +} + + +const val POSE_ARRAY_SIZE = 56 +const val CROSS_ARM_RATIO_THRESHOLD = 0.10f +val CROSS_ARM_ANGLE_THRESHOLD = listOf(10.0f, 170.0f) +const val FALLDOWN_TILT_RATIO = 0.80f +const val FALLDOWN_TILT_ANGLE = 15.0f +const val BODY_TILT_RATIO = 0.30f + + +class HpeClassification( + private val pose_info: FloatArray?, + private val cross_ratio_threshold: Float = CROSS_ARM_RATIO_THRESHOLD, + private val cross_angle_threshold: List = CROSS_ARM_ANGLE_THRESHOLD, + private val falldown_tilt_ratio: Float = FALLDOWN_TILT_RATIO, + private val falldown_tilt_angle: Float = FALLDOWN_TILT_ANGLE, + private val body_tilt_ratio: Float = BODY_TILT_RATIO +) { + private var cross_line1_point1: PointF? = null + private var cross_line1_point2: PointF? = null + private var cross_line2_point1: PointF? = null + private var cross_line2_point2: PointF? = null + + private var kp_rhip: PointF? = null + private var kp_lhip: PointF? = null + private var kp_rsldr: PointF? = null + private var kp_lsldr: PointF? = null + private var kp_rft: PointF? = null + private var kp_lft: PointF? = null + + private var cross_point: PointF? = null + private var cross_angle: Float? = null + + private var overturn_upper = false + private var overturn_lower = false + private var badly_tilt = false + private var body_tilt_angle: List? = null + + init { + if (pose_info != null && pose_info.size == POSE_ARRAY_SIZE) { + cross_line1_point1 = PointF(pose_info[KeyPoint.RIGHT_ELBOW.x], pose_info[KeyPoint.RIGHT_ELBOW.y]) + cross_line1_point2 = PointF(pose_info[KeyPoint.RIGHT_WRIST.x], pose_info[KeyPoint.RIGHT_WRIST.y]) + cross_line2_point1 = PointF(pose_info[KeyPoint.LEFT_ELBOW.x], pose_info[KeyPoint.LEFT_ELBOW.y]) + cross_line2_point2 = PointF(pose_info[KeyPoint.LEFT_WRIST.x], pose_info[KeyPoint.LEFT_WRIST.y]) + + if ((cross_line1_point1!!.x > 0 || cross_line1_point1!!.y > 0) && + (cross_line1_point2!!.x > 0 || cross_line1_point2!!.y > 0) && + (cross_line2_point1!!.x > 0 || cross_line2_point1!!.y > 0) && + (cross_line2_point2!!.x > 0 || cross_line2_point2!!.y > 0)) { + _detect_cross_arms() + } + + kp_rhip = PointF(pose_info[KeyPoint.RIGHT_HIP.x], pose_info[KeyPoint.RIGHT_HIP.y]) + kp_lhip = PointF(pose_info[KeyPoint.LEFT_HIP.x], pose_info[KeyPoint.LEFT_HIP.y]) + kp_rsldr = PointF(pose_info[KeyPoint.RIGHT_SHOULDER.x], pose_info[KeyPoint.RIGHT_SHOULDER.y]) + kp_lsldr = PointF(pose_info[KeyPoint.LEFT_SHOULDER.x], pose_info[KeyPoint.LEFT_SHOULDER.y]) + kp_rft = PointF(pose_info[KeyPoint.RIGHT_FEET.x], pose_info[KeyPoint.RIGHT_FEET.y]) + kp_lft = PointF(pose_info[KeyPoint.LEFT_FEET.x], pose_info[KeyPoint.LEFT_FEET.y]) + _detect_falldown() + } + } + + fun is_cross_arms(): Boolean { + if (cross_point != null && cross_angle != null) { + if (cross_angle!! > cross_angle_threshold[0] && cross_angle!! < cross_angle_threshold[1]) { + return true + } + } + + return false + } + + fun is_falldown(is_working_on: Boolean = true): Boolean { + val result = (overturn_lower || overturn_upper || badly_tilt) + var tilt_angle = false + + if (body_tilt_angle != null) { + if (body_tilt_angle!![0] < falldown_tilt_angle || body_tilt_angle!![1] < falldown_tilt_angle) { + tilt_angle = true + } + } + + return (result || (tilt_angle && is_working_on)) + } + + fun get_hpe_type(is_working_on: Boolean = true): Int { + var result = 0x0000 + + if (is_falldown(is_working_on)) { + result = result or HPETypeMask.FALLDOWN + } + if (is_cross_arms()) { + result = result or HPETypeMask.CROSSARM + } + + return result + } + + fun get_hpe_level(is_working_on: Boolean = true): Int { + var result = 0 + + if (is_falldown(is_working_on) && result < HPELevel.FALLDOWN) { + result = HPELevel.FALLDOWN + } + + if (is_cross_arms() && result < HPELevel.CROSSARM) { + result = HPELevel.CROSSARM + } + + return result + } + + private fun _detect_falldown() { + val threshold = 0.3f + + overturn_upper = false + overturn_lower = false + badly_tilt = false + body_tilt_angle = null + + // 1. 상체 뒤집힘. + if (pose_info != null && + (pose_info[KeyPoint.LEFT_HIP.c] > threshold && pose_info[KeyPoint.LEFT_SHOULDER.c] > threshold && + pose_info[KeyPoint.RIGHT_HIP.c] > threshold && pose_info[KeyPoint.LEFT_SHOULDER.c] > threshold) + ) { + if (kp_lhip!!.y < kp_lsldr!!.y && kp_rhip!!.y < kp_rsldr!!.y) { + overturn_upper = true + } + } + + // 2. 하체 뒤집힘 + if (pose_info != null && + (pose_info[KeyPoint.LEFT_HIP.c] > threshold && pose_info[KeyPoint.LEFT_FEET.c] > threshold && + pose_info[KeyPoint.RIGHT_HIP.c] > threshold && pose_info[KeyPoint.RIGHT_FEET.c] > threshold) + ) { + if (kp_lhip!!.y > kp_lft!!.y && kp_rhip!!.y > kp_rft!!.y) { + overturn_lower = true + } + } + + // 3. 신체 기울어짐 (좌우 어깨, 좌우 발) + if (pose_info != null && + (pose_info[KeyPoint.LEFT_SHOULDER.c] > threshold && pose_info[KeyPoint.RIGHT_SHOULDER.c] > threshold && + pose_info[KeyPoint.LEFT_FEET.c] > threshold && pose_info[KeyPoint.RIGHT_FEET.c] > threshold) + ) { + val xmax = maxOf(kp_lsldr!!.x, kp_rsldr!!.x, kp_lft!!.x, kp_rft!!.x) + val ymax = maxOf(kp_lsldr!!.y, kp_rsldr!!.y, kp_lft!!.y, kp_rft!!.y) + val xmin = minOf(kp_lsldr!!.x, kp_rsldr!!.x, kp_lft!!.x, kp_rft!!.x) + val ymin = minOf(kp_lsldr!!.y, kp_rsldr!!.y, kp_lft!!.y, kp_rft!!.y) + val width = abs(xmax - xmin) + val height = abs(ymax - ymin) + + if ((width > 1e-4) && ((height / width) < falldown_tilt_ratio)) { + badly_tilt = true + } + } + + // 4. 상체 기울어짐 (좌우 어깨, 좌우 엉덩이) + if (pose_info != null && + (pose_info[KeyPoint.LEFT_SHOULDER.c] > threshold && pose_info[KeyPoint.RIGHT_SHOULDER.c] > threshold && + pose_info[KeyPoint.LEFT_HIP.c] > threshold && pose_info[KeyPoint.RIGHT_HIP.c] > threshold) + ) { + val angle_left = _get_cross_angle(kp_lsldr!!, kp_rhip!!, kp_lhip!!) + val angle_right = _get_cross_angle(kp_lsldr!!, kp_rhip!!, kp_lhip!!) + body_tilt_angle = listOf(angle_left, angle_right) + + val xmaxBody = maxOf(kp_lsldr!!.x, kp_rsldr!!.x, kp_lhip!!.x, kp_rhip!!.x) + val ymaxBody = maxOf(kp_lsldr!!.y, kp_rsldr!!.y, kp_lhip!!.y, kp_rhip!!.y) + val xminBody = minOf(kp_lsldr!!.x, kp_rsldr!!.x, kp_lhip!!.x, kp_rhip!!.x) + val yminBody = minOf(kp_lsldr!!.y, kp_rsldr!!.y, kp_lhip!!.y, kp_rhip!!.y) + val widthBody = abs(xmaxBody - xminBody) + val heightBody = abs(ymaxBody - yminBody) + + if ((heightBody > 1e-4) && ((widthBody / heightBody) < body_tilt_ratio)) { + body_tilt_angle = null + } + } + } + + private fun _detect_cross_arms() { + cross_point = null + cross_angle = null + + if (cross_line1_point1 != null && cross_line1_point2 != null && + cross_line2_point1 != null && cross_line2_point2 != null) { + + // 손목이 팔꿈치 아래에 있는 경우 제외 + if (cross_line1_point1!!.y < cross_line2_point2!!.y && + cross_line2_point1!!.y < cross_line2_point2!!.y) { + return + } + + val line1_ccw = _ccw(cross_line1_point1!!, cross_line1_point2!!, cross_line2_point1!!) * + _ccw(cross_line1_point1!!, cross_line1_point2!!, cross_line2_point2!!) + val line2_ccw = _ccw(cross_line2_point1!!, cross_line2_point2!!, cross_line1_point1!!) * + _ccw(cross_line2_point1!!, cross_line2_point2!!, cross_line1_point2!!) + + if (line1_ccw < 0 && line2_ccw < 0) { + cross_point = _get_cross_point() + cross_angle = _get_cross_angle(cross_line1_point1!!, cross_line2_point2!!, cross_point!!) + + Log.d(LOG_TAG_DEBUG, "cross_point: $cross_point, cross_angle: $cross_angle") + } + } + } + + private fun _ccw(p1: PointF, p2: PointF, p3: PointF): Int { + val cross_product = ((p2.x - p1.x) * (p3.y - p1.y)) - ((p3.x - p1.x) * (p2.y - p1.y)) + + if (cross_product > 0) + return 1 // 반시계방향 (counter clockwise) + else if (cross_product < 0) + return -1 // 시계방향 (clockwise) + + return 0 // 평행 (collinear) + } + + private fun _get_cross_point(): PointF { + val x1 = cross_line1_point1!!.x + val y1 = cross_line1_point1!!.y + val x2 = cross_line1_point2!!.x + val y2 = cross_line1_point2!!.y + val x3 = cross_line2_point1!!.x + val y3 = cross_line2_point1!!.y + val x4 = cross_line2_point2!!.x + val y4 = cross_line2_point2!!.y + + val cx = ((x1 * y2 - x2 * y1) * (x3 - x4) - (x1 - x2) * (x3 * y4 - x4 * y3)) / + ((x1 - x2) * (y3 - y4) - (y1 - y2) * (x3 - x4)) + val cy = ((x1 * y2 - x2 * y1) * (y3 - y4) - (y1 - y2) * (x3 * y4 - x4 * y3)) / + ((x1 - x2) * (y3 - y4) - (y1 - y2) * (x3 - x4)) + + return PointF(cx, cy) + } + + fun get_cross_point(): PointF { + return cross_point ?: PointF(0f, 0f) + } + + private fun _get_cross_angle(p1: PointF, p2: PointF, cp:PointF): Float { + var radian = atan2((p1.y - cp.y), (p1.x - cp.x)) - atan2((p2.y - cp.y), (p2.x - cp.x)) + + radian = abs(radian * 180f / PI.toFloat()) + + if (radian > 180) { + radian = 360 - radian + } + + return radian + } +} \ No newline at end of file diff --git a/app/src/main/java/com/a2d2/mobilitygateway/MQTTManager.kt b/app/src/main/java/com/a2d2/mobilitygateway/MQTTManager.kt new file mode 100644 index 0000000..f656fee --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/MQTTManager.kt @@ -0,0 +1,189 @@ +package com.a2d2.mobilitygateway + +import android.util.Log +import androidx.appcompat.app.AlertDialog +import info.mqtt.android.service.MqttAndroidClient +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import org.eclipse.paho.client.mqttv3.IMqttActionListener +import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken +import org.eclipse.paho.client.mqttv3.IMqttToken +import org.eclipse.paho.client.mqttv3.MqttException +import org.eclipse.paho.client.mqttv3.MqttCallback +import org.eclipse.paho.client.mqttv3.MqttClient +import org.eclipse.paho.client.mqttv3.MqttConnectOptions +import org.eclipse.paho.client.mqttv3.MqttMessage +import org.json.JSONObject +import java.time.LocalDateTime +import java.util.Queue +import java.util.LinkedList +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + + +@androidx.media3.common.util.UnstableApi // FERMAT(HSJ100): LOCAL MODULE +class MQTTManager(private val engine: EngineActivity, private val ipaddr: String, private val port: Int) { + private val mqttClient: MqttAndroidClient + private val util: Util = Util() + private val biQueue: Queue = LinkedList() + private val mqttLock = Mutex() +// private val biQueue = LinkedBlockingQueue(100) + + var connectStatus = 0 //0:시작전 ,1:connect 완료, 2:connect 실패 + + init { + val uri = "tcp://$ipaddr:$port" + mqttClient = MqttAndroidClient(engine.applicationContext, uri, MqttClient.generateClientId()) + } + + private fun log(level: String = "", message: String, logcat: Boolean = true) { + engine.log(level, message, logcat) + } + + fun checkConnection(): Boolean { + return mqttClient.isConnected() + } + + fun connect(username: String, password: String) { + val mqttConnectOptions = MqttConnectOptions() + mqttConnectOptions.isAutomaticReconnect = true + if(username.isNotEmpty() && password.isNotEmpty()) { + mqttConnectOptions.userName = username + mqttConnectOptions.password = password.toCharArray() + } + mqttClient.setCallback(mqttCallback) + + // 연결체크 비동기 방식 + mqttClient.connect(mqttConnectOptions, null, object : IMqttActionListener{ + override fun onSuccess(asyncActionToken: IMqttToken?) { + log("info", "mqtt connected to '$ipaddr:$port'") + } + + override fun onFailure(asyncActionToken: IMqttToken?, exception: Throwable?) { + log("error", "MQTT CONNECTION FAIL: ${exception?.message}, ${exception?.cause}") + engine.finish() + } + }) + } + + private val mqttCallback = object : MqttCallback { + override fun connectionLost(cause: Throwable?) { + log("error", "mqtt disconnected${if(cause != null) ":: $cause" else ""}") + } + + // 구독한 topic에서 메시지가 도착한 경우 + override fun messageArrived(topic: String?, message: MqttMessage?) { + runBlocking { + mqttLock.withLock { //kotlin mutax queue + biQueue.offer(message.toString()) + } + } + } + + // 메시지를 publish 한 경우 + override fun deliveryComplete(token: IMqttDeliveryToken?) { + val message = token?.message.toString() + val length = message.length + + Log.d(LOG_TAG_DEBUG, "mqtt message published ($length bytes) ::") + + if(length > 300) { + Log.d(LOG_TAG_DEBUG, message.take(7) + " ... " + message.takeLast(7)) + } + else { + if(util.isJson(message)) { + val jsonObject = JSONObject(message) + Log.d(LOG_TAG_DEBUG, jsonObject.toString(4)) + } + else { + Log.d(LOG_TAG_DEBUG, message) + } + } + } + } + + fun subscribe(topic: String) { + resetQueue() + mqttClient.subscribe(topic, MQTT_QOS) + } + + fun unSubscribe(topic: String){ + mqttClient.unsubscribe(topic) + resetQueue() + } + + private fun resetQueue(){ + biQueue.clear() + } + + fun getQueneMessages(): List { + var messasgeList = mutableListOf() + runBlocking { + mqttLock.withLock { //kotlin mutax queue + while (biQueue.isNotEmpty()){ + var message: String? = biQueue.peek() ?: break + if (message != null) { + messasgeList.add(message) + biQueue.poll() + } + } + } + } + return messasgeList + } + + fun publishMessage(topic: String, text: String) { + val msg = MqttMessage(text.toByteArray()).apply { + this.qos = MQTT_QOS + } + + mqttClient.publish(topic, msg) + } + + fun publishMessage(topic: String, message: HashMap) { + val msg = JSONObject(message).toString() + + publishMessage(topic, msg) + } + + fun publishMessage(topic: String, message: MQTTResponse) { + val msg = Json.encodeToString(message) + + publishMessage(topic, msg) + } + + fun publishMessage(topic: String, message: MQTTResponseFr) { + val msg = Json.encodeToString(message) + + publishMessage(topic, msg) + } + + fun publishMessage(topic: String, message: MQTTResponseWD) { + val msg = Json.encodeToString(message) + + publishMessage(topic, msg) + } + + fun publishError(topic: String, message: String, request: JsonObject? = null, result: MQTTResponseResult? = null) { + val msg = MQTTResponse( + datetime = LocalDateTime.now().format(MQTT_DATETIME_FORMAT), + status = RESPONSE_STATUS.ERROR, + message = message, + requests = request, + result = result?: MQTTResponseResult( + detect = null, + image_data = null + ) + ) + + publishMessage(topic, msg) + } + + fun disconnect() { + + mqttClient.disconnect() +// mqttClient.close() //TODO(JWKIM): close 사용시 앱이 죽어버림 + } +} \ No newline at end of file diff --git a/app/src/main/java/com/a2d2/mobilitygateway/ObjectDetect.kt b/app/src/main/java/com/a2d2/mobilitygateway/ObjectDetect.kt new file mode 100644 index 0000000..0fc5b10 --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/ObjectDetect.kt @@ -0,0 +1,463 @@ +package com.a2d2.mobilitygateway + +import android.graphics.Bitmap +import android.graphics.Rect +import android.net.Uri +import android.util.Log +import androidx.media3.common.util.UnstableApi +import com.a2d2.mobilitygateway.objectDetect.OnnxManager +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.boolean +import kotlinx.serialization.json.int +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import java.time.LocalDateTime +import java.util.Timer +import java.util.TimerTask +import com.alexvas.rtsp.widget.RtspSurfaceView +import com.a2d2.mobilitygateway.objectDetect.BitmapDraw + + +@UnstableApi +open class ObjectDetectWD( + private val onnxManager: OnnxManager, + private val mqttManager: MQTTManager?, + private val request: JsonObject, + private val engine: EngineActivity +) { + protected val util: Util = Util() + protected open val mqttTopic = TOPIC.WD + + protected var limitTimer: Timer? = null + protected var eventStop = false + protected var eventTimeout = false + protected var eventComplete = false + + protected val requestStruct = RequestJson() + private val detectedInfo = LinkedHashMap() // detect 해야할 class 정보 + protected val classInfo = HashMap() // 전체 class info + protected var currentStatus: String = RESPONSE_STATUS.START + + protected lateinit var coroutineJob: Job + protected val coroutineScope = CoroutineScope(Dispatchers.Default) + + private var alertLevel: Int = ALERT_LEVEL.NORMAL + + private var odInferenceTime : Long? = null + private var poseInferenceTime : Long? = null + private var totalProcesstime : Long? = null + + private var bi_message: MQTTResponseBI? = null + private var hpe_detect_list: ArrayList? = null + private var hpe_message: MQTTResponseHpe? = null + private var od_message: MQTTResponseObject? = null + + private var currentRtspUrl: String? = null + private var rtspConnectCount : Int = 2 + private var rtspBrokenPipeFlag : Boolean = false + + private var bitmapImage: Bitmap? = null + + // FERMAT(HSJ100): RTSP_BLANK_FRAME + open var rtspStatusListener = object: RtspSurfaceView.RtspStatusListener { + override fun onRtspStatusConnecting() {} + override fun onRtspStatusConnected() {} + override fun onRtspStatusDisconnecting() {} + override fun onRtspStatusDisconnected() {} + override fun onRtspStatusFailedUnauthorized() {} + override fun onRtspStatusFailed(message: String?) {} + override fun onRtspFirstFrameRendered() {} + override fun onRtspBlankFrame() { + log("warning", "Blank pipe") + rtspBrokenPipeFlag = true + } + override fun onRtspClientStarted() { + rtspBrokenPipeFlag = false + rtspConnectCount = 2 + } + override fun onRtspClientStopped() { + engine.rtspPreview.stop() + if (rtspBrokenPipeFlag) { + if (rtspConnectCount > 0){ + rtspConnectCount-- + engine.rtspPreview.start(requestVideo = true, requestAudio = false) + } else{ + rtspBrokenPipeFlag = false + rtspConnectCount = 2 + } + } else{ + rtspConnectCount = 2 + } + } + } + + class RequestJson { + var limit_time_min: Int = 0 + var report_unit: Boolean = false + var crop_image: Boolean = false + var snap_shot: Boolean = false + var detect_list = HashMap() + } + + init { + var idx = 0 + onnxManager.classes.forEach { + classInfo.put(idx++, it) + } + } + + protected fun log(level: String = "", message: String, logcat: Boolean = true) { + engine.log(level, message, logcat) + } + + protected open fun setRequestInfo(request: JsonObject) { + val targets = request["targets"]?.jsonObject + requestStruct.limit_time_min = request["limit_time_min"]?.jsonPrimitive?.int?: 0 + requestStruct.report_unit = request["report_unit"]?.jsonPrimitive?.boolean?: false + requestStruct.crop_image = targets?.get("crop_image")?.jsonPrimitive?.boolean?: false + requestStruct.snap_shot = targets?.get("snap_shot")?.jsonPrimitive?.boolean?: false + targets?.get("detect_list")?.jsonArray?.forEach { + val class_id = it.jsonObject["class_id"]!!.jsonPrimitive.int + val class_name = it.jsonObject["class_name"]?.jsonPrimitive?.content?: "" + val useable = it.jsonObject["useable"]?.jsonPrimitive?.boolean + + if (useable == true && class_id in classInfo.keys) { + requestStruct.detect_list.put(class_id, class_name) + if (class_id in classInfo.keys) { + // 탐지해야 할 class update. name은 앱이 가진 class name을 사용한다. + detectedInfo.put(classInfo[class_id]!!, null) + } + } + } + } + + open fun start() { + try { + // start 상태 전송 + publishMessage(RESPONSE_STATUS.START) + + setRequestInfo(request) + + // timer start + val timerTask = object : TimerTask() { + override fun run() { + log("warning", "timer expired") + eventTimeout = true + } + } + if (requestStruct.limit_time_min > 0) { + limitTimer = Timer() + limitTimer?.schedule(timerTask, requestStruct.limit_time_min * 60 * 1000L) + } + + if (use_camera){ + + } + else{ + currentRtspUrl = RTSP_BASE_URL + + // rtsp client 구동 + engine.rtspPreview.stop() + engine.rtspPreview.init(Uri.parse(currentRtspUrl), RTSP_USER, RTSP_PW) + engine.rtspPreview.setStatusListener(rtspStatusListener) + engine.rtspPreview.start(requestVideo = true, requestAudio = false) + } + coroutineJob = coroutineScope.launch { + var imageBitmap: Bitmap? = null + + // ready 상태 전송 + publishMessage(RESPONSE_STATUS.READY) + + while (isActive) { + if (eventStop) { + publishMessage(RESPONSE_STATUS.STOP) + break + } else if (eventTimeout) { + publishMessage(RESPONSE_STATUS.TIMEOUT) + break + } else if (currentStatus == RESPONSE_STATUS.ERROR) { + break + } + + // get bitmap +// imageBitmap = null + imageBitmap = if (use_camera) engine.currentImageData else engine.getSnapshot() + + if (imageBitmap == null) + continue + + processImage(imageBitmap) + + if (alertLevel != ALERT_LEVEL.NORMAL) { +// val snapshot = if (requestStruct.snap_shot) imageBitmap else null + val snapshot = if (requestStruct.snap_shot) bitmapImage else null + publishMessage(RESPONSE_STATUS.DETECT, snapshot = snapshot) + alertLevel = ALERT_LEVEL.NORMAL + } + } + engine.rtspPreview.stop() + finish() + } + } + catch (e: Exception) { + publishMessage(RESPONSE_STATUS.ERROR, e.toString()) + finish() + } + } + + fun getIsActive(): Boolean { + return (::coroutineJob.isInitialized && coroutineJob.isActive) + } + + fun getIsComplete(): Boolean { + return (coroutineJob.isCompleted) + } + + fun getIsCancelled(): Boolean { + return (coroutineJob.isCancelled) + } + + fun stop() { + eventStop = true + } + + fun resetStop(){ + eventStop = false + } + + private suspend fun processImage(bitmap: Bitmap) { + /* + inference 된 결과를 받아서 처리하는 부분 + */ + withContext(Dispatchers.Default) { + try { + totalProcesstime = null + var processStartTime = System.currentTimeMillis() + + odInferenceTime = null + var results_od = arrayListOf() + if (USE_OD_MODEL){ + var odStartTime = System.currentTimeMillis() + results_od = onnxManager.inferenceOD(bitmap) + var odEndTime = System.currentTimeMillis() + odInferenceTime = odEndTime - odStartTime + } + + poseInferenceTime = null + var results_pose = arrayListOf() + if (USE_POSE_MODEL){ + var poseStartTime = System.currentTimeMillis() + results_pose = onnxManager.inferencePose(bitmap) + var poseEndTime = System.currentTimeMillis() + poseInferenceTime = poseEndTime - poseStartTime + } + + process_od(results_od, bitmap) + process_pose(results_pose, bitmap) + + bitmapImage = null + val bitmapDraw = BitmapDraw() + bitmapDraw.onLabelDraw(bitmap,results_od,results_pose) + bitmapImage = bitmapDraw.getBitmap() + +// engine.setAlertView(alertLevel) + var processEndTime = System.currentTimeMillis() + totalProcesstime = processEndTime - processStartTime + } + catch (e: Exception) { + publishMessage(RESPONSE_STATUS.ERROR, e.toString()) + } + } + } + + private fun process_od(results: ArrayList, bitmap: Bitmap) { + /* + object detect 된 결과를 메세지 변환 + */ + val cnt = Array(WD_WARNING_DETECT_CLASS_LIST.size) { 0 } + + od_message = MQTTResponseObject( + detect_type = "danger1", + detect = ArrayList(), + inference_time = "$odInferenceTime ms" + ) + + results.forEach { result -> + var cropImage: String? = null + val rect = util.getRectFromBbox(bitmap, result.bbox) + + try { + if (requestStruct.crop_image) { + val cropBitmap = util.cropBitmapUsingBbox(bitmap, result.bbox) + + cropImage = util.encodeToBase64(cropBitmap) + } + } + catch (e: Exception) { + Log.d(LOG_TAG_DEBUG, "crop error:: $e") + cropImage = null + } + + WD_WARNING_DETECT_CLASS_LIST.forEachIndexed { index, className -> + if (classInfo[result.classIndex] == className) { + cnt[index]++ + od_message!!.detect.add( + MQTTResponseDetect( + class_id = result.classIndex, + class_name = className, + confidence = result.score, + bbox = arrayListOf(rect.left, rect.top, rect.right, rect.bottom), + image = cropImage + ) + ) + } + } + } + + var warningCnt = 0 + WD_WARNING_DETECT_CLASS_LIST.forEachIndexed { index, s -> + if (cnt[index] > 0) { + warningCnt++ + } + } + + if (warningCnt >= WD_WARNING_DETECT_CLASS_LIST.size) { + setAlertLevel(ALERT_LEVEL.WARNING) + } + else { + od_message = null + } + + engine.updateRectView(results) + } + + private fun process_pose(results: ArrayList, bitmap: Bitmap) { + /* + pose detect 된 결과를 메세지 변환 + */ + hpe_detect_list = ArrayList() + hpe_message = null + + results.forEach { pose_info -> + val hpe = HpeClassification(pose_info) + val hpeLevel = hpe.get_hpe_level() + val hpeType = hpe.get_hpe_type() + var cropImage: String? = null + + if (hpeType != HPETypeMask.NORMAL) { + val factor_w = CAMERA_IMAGE_WIDTH.toFloat() / TRAIN_IMAGE_SIZE.toFloat() + val factor_h = CAMERA_IMAGE_HEIGHT.toFloat() / TRAIN_IMAGE_SIZE.toFloat() + val bboxRect = Rect( + (pose_info[0]*factor_w).toInt(), + (pose_info[1]*factor_h).toInt(), + (pose_info[2]*factor_w).toInt(), + (pose_info[3]*factor_h).toInt() + ) + + try { + if (requestStruct.crop_image) { + val cropBitmap = util.cropBitmapUsingRect(bitmap, bboxRect) + + cropImage = util.encodeToBase64(cropBitmap) + } + } + catch (e: Exception) { + Log.d(LOG_TAG_DEBUG, "crop error:: $e") + cropImage = null + } + + setAlertLevel(ALERT_LEVEL.DANGER) + + hpe_detect_list!!.add( + MQTTResponseHpeDetect( + class_id = 0, + class_name = "person", + confidence = pose_info[4], + bbox = arrayListOf(bboxRect.left, bboxRect.top, bboxRect.right, bboxRect.bottom), + image = cropImage, + pose_type = hpeType, + pose_level = hpeLevel + ) + ) + + if (hpe.is_cross_arms()) { + val cross_point = hpe.get_cross_point() + Log.d(LOG_TAG_DEBUG, "detect cross arm (%.2f, %.2f)".format(cross_point.x, cross_point.y)) + } + + if (hpe.is_falldown()) { + Log.d(LOG_TAG_DEBUG, "detect falldown") + } + } + } + + if (hpe_detect_list?.isEmpty() == true) { + hpe_detect_list = null + } + else{ + hpe_message = MQTTResponseHpe(detect =hpe_detect_list, inference_time = "$poseInferenceTime ms") + } + + engine.updatePoseView(results) + } + + private fun setAlertLevel(level: Int) { + if (alertLevel < level) + alertLevel = level + } + + protected open fun publishMessage(status: String, message: String? = null, snapshot: Bitmap? = null) { + /* + MQTT 메세지 전송 + */ + log("info", javaClass.simpleName + " STATUS $status".uppercase()) + currentStatus = status + + val mqttResponse = MQTTResponseWD( + datetime = LocalDateTime.now().format(MQTT_DATETIME_FORMAT), + status = currentStatus, + message = message, + requests = request, + result = MQTTResponseWDResult( + ri_info = null, + bi = bi_message, + hpe = hpe_message, + od = od_message, + image_data = util.encodeToBase64(snapshot, forceScale = true,), + process_time = if (totalProcesstime != null) {"$totalProcesstime ms"} else { null } + ) + ) + mqttManager?.publishMessage(mqttTopic, mqttResponse) + } + + private fun finish() { + /* + 종료처리 + */ + if (::coroutineJob.isInitialized) + coroutineJob.cancel() + + eventStop = false + eventTimeout = false + eventComplete = false + + limitTimer?.cancel() + + engine.updateRectView() + engine.updatePoseView() + + alertLevel = ALERT_LEVEL.NORMAL +// engine.setAlertView(alertLevel) + bi_message = null + hpe_detect_list = null + od_message = null + } + +} \ No newline at end of file diff --git a/app/src/main/java/com/a2d2/mobilitygateway/RESTManager.kt b/app/src/main/java/com/a2d2/mobilitygateway/RESTManager.kt new file mode 100644 index 0000000..57e5a38 --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/RESTManager.kt @@ -0,0 +1,372 @@ +package com.a2d2.mobilitygateway + +import android.os.Handler +import android.os.Looper +import android.os.Message +import android.util.Log +import com.a2d2.mobilitygateway.objectDetect.OnnxManager +import fi.iki.elonen.NanoHTTPD +import fi.iki.elonen.NanoHTTPD.Response.IStatus +import fi.iki.elonen.NanoHTTPD.Response.Status +import fi.iki.elonen.router.RouterNanoHTTPD +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject +import org.json.JSONObject +import java.net.NetworkInterface + + +// singlton 형태로 사용할 것. +@androidx.media3.common.util.UnstableApi // FERMAT(HSJ100): LOCAL MODULE +class RESTManager(private val engine: EngineActivity, private val port: Int) : RouterNanoHTTPD(port) { + companion object { + val ipAddr = "0.0.0.0" // FERMAT HSJ100: FIX CORS + + @Volatile private var server: RESTManager? = null + fun getServer(engine: EngineActivity, port: Int): RESTManager { + return server ?: synchronized(this) { + server ?: RESTManager(engine, port).also { + server = it + } + } + } + + private fun getIpAdderss() : String { + val en = NetworkInterface.getNetworkInterfaces() + while (en.hasMoreElements()) { + val intf = en.nextElement() + val enumIpAddr = intf.inetAddresses + + while (enumIpAddr.hasMoreElements()) { + val inetAddress = enumIpAddr.nextElement() + + if (inetAddress.hostAddress != null) { +// if (inetAddress.isLoopbackAddress) { +// Log.d(LOG_TAG_DEBUG, intf.displayName + "(loopback) | " + inetAddress.hostAddress) +// } else { +// Log.d(LOG_TAG_DEBUG, intf.displayName + " | " + inetAddress.hostAddress) +// } + if (!inetAddress.isLoopbackAddress && inetAddress.isSiteLocalAddress) { + Log.d(LOG_TAG_DEBUG, "local address: ${intf.displayName}, ${inetAddress.hostAddress}") + return inetAddress.hostAddress!! + } + } + } + } + return "can't find ip addr" + } + } + + private fun log(level: String = "", message: String, logcat: Boolean = true) { + engine.log(level, message, logcat) + } + + fun addRoute(url: String, routeHandler: Class<*>, looperHandler: Handler?) { + super.addRoute(url, routeHandler, looperHandler) + } + + fun addRoute(url: String, routeHandler: Class<*>, looperHandler: Handler?, engine: EngineActivity) { + super.addRoute(url, routeHandler, looperHandler, engine) + } + + fun startServer() { + start() + log("info", "start REST server with $ipAddr:$port") + } + + fun stopServer() { + stop() + log("info", "stop REST server") + } +} + + +// 기본 http server 테스트 예제 +class HTTPManager(private val handler: Handler) : NanoHTTPD(7777) { + /* + ### 기본 http server 테스트 예제 + + 아래와 같이 사용 + ``` + val http = HTTPManager(object : Handler(Looper.getMainLooper()) { + override fun handleMessage(msg: Message) { + var uri = "" + lateinit var data: RestTestRequest + (msg.obj as Map<*, *>).firstNotNullOf { + uri = it.key as String + data = it.value as RestTestRequest + } + Log.d(LOG_TAG_DEBUG, "DATA:: $data") + super.handleMessage(msg) + } + }) + ''' + */ + + init { + start(SOCKET_READ_TIMEOUT, false) + } + + override fun serve(session: IHTTPSession): Response { + Log.d(LOG_TAG_DEBUG, "URI:: ${session.uri}") + Log.d(LOG_TAG_DEBUG, "method:: ${session.method}") + + val hashMap = HashMap() + session.parseBody(hashMap) + + val receivedBody = if(hashMap.isNotEmpty()) hashMap["postData"] else session.queryParameterString + if (receivedBody != null) { + val data = Json.decodeFromString(receivedBody) + val msg: Message = Message().apply { + this.obj = mapOf(session.uri to data) + } + handler.sendMessage(msg) + } + + return newFixedLengthResponse("THIS IS TEST SERVER") + } +} + +class TitleRestHandler : RouterNanoHTTPD.DefaultHandler() { + override fun getText(): String { + return "

Mobility Gateway for Android

" + } + + override fun getMimeType(): String { + return "text/html" + } + + override fun getStatus(): IStatus { + return Status.OK + } +} + + +open class BaseRestHandler : RouterNanoHTTPD.GeneralHandler() { + private var looperHandler: Handler? = null + + open val topic = TOPIC.TEST + + override fun post( + uriResource: RouterNanoHTTPD.UriResource?, + urlParams: MutableMap?, + session: NanoHTTPD.IHTTPSession? + ): NanoHTTPD.Response { + Log.d(LOG_TAG_DEBUG, "ENTER ${this::class.simpleName} POST from uri '${session?.uri}'") + + try { + // utf-8 encoding 처리, utf-8 설정이 없는 경우 한글 깨짐. + if (session?.headers?.get("content-type") != null && + session.headers?.get("content-type")?.contains("charset=UTF-8", true) == false) { + session.headers["content-type"] += "; charset=UTF-8" + } + + looperHandler = uriResource?.initParameter(Handler::class.java) + + val hashMap = HashMap().also { + session?.parseBody(it) + } + val body = if(hashMap.isNotEmpty()) hashMap["postData"] else session?.queryParameterString + + checkValid(session?.uri, body) + + val msg: Message = Message().apply { + this.obj = listOf(session?.uri?:"", body?:"") + } + + // (uri, body) 형태로 전달 + looperHandler?.sendMessage(msg) + + return makeResponse() + } + catch (e: Exception) { + Log.e(LOG_TAG_DEBUG, "REST POST ERROR at ${session?.uri}") + return makeResponse(false, e.message) + } + } + + open fun checkValid(uri: String?, body: String?) { + if (uri.isNullOrBlank()) { + throw Exception("rest api uri error") + } + } + + private fun makeResponse(result: Boolean = true, errMsg: String? = null): NanoHTTPD.Response { + val map = LinkedHashMap() + + map["result"] = result + map["error"] = errMsg + map["topic"] = topic + + return NanoHTTPD.newFixedLengthResponse( + if (result) Status.OK else Status.BAD_REQUEST, + "application/json", + JSONObject(map).toString() + ) + } +} + + +open class JsonCheckRestHandler : BaseRestHandler() { + private val util: Util = Util() + + override fun checkValid(uri: String?, body: String?) { + super.checkValid(uri, body) + + if (body.isNullOrBlank()) { + throw Exception("body is empty") + } + + if (util.isNotJson(body)) { + throw Exception("body is not json format") + } + } +} + + +class ConRestHandler : JsonCheckRestHandler() { + override val topic = TOPIC.CON +} + + +class FrRestHandler : JsonCheckRestHandler() { + override val topic = TOPIC.FR +} + +class WDRestHandler : JsonCheckRestHandler() { + override val topic = TOPIC.WD +} + +class BIRestHandler : JsonCheckRestHandler() { + override val topic = TOPIC.BI +} + + +class ClassInfoRestHandler : RouterNanoHTTPD.GeneralHandler() { + override fun get( + uriResource: RouterNanoHTTPD.UriResource?, + urlParams: MutableMap?, + session: NanoHTTPD.IHTTPSession? + ): NanoHTTPD.Response { + Log.d(LOG_TAG_DEBUG, "ENTER ${this::class.simpleName} GET from uri '${session?.uri}'") + + try { + val engine = uriResource?.initParameter(1, EngineActivity::class.java) + val modelManager = engine?.getModel() + + return makeResponse(modelManager = modelManager) + } + catch (e: Exception) { + Log.e(LOG_TAG_DEBUG, "REST GET ERROR at ${session?.uri}") + return makeResponse(false, e.message, null) + } + } + + private fun makeResponse(result: Boolean = true, errMsg: String? = null, modelManager: OnnxManager?): NanoHTTPD.Response { + val map = LinkedHashMap() + val classInfo = LinkedHashMap() + var idx = 0 + + try { + if (!result && errMsg != null) { + throw Exception(errMsg) + } + + modelManager?.classes?.forEach { + classInfo["${idx++}"] = it + } + + if (idx == 0) { + throw Exception("not found class info") + } + + map["result"] = (idx > 0) + map["error"] = null + map["info"] = classInfo + } + catch (e: Exception) { + map["result"] = false + map["error"] = e.message + map["info"] = null + } + + return NanoHTTPD.newFixedLengthResponse( + if (map["result"] == true) Status.OK else Status.INTERNAL_ERROR, + "application/json", + JSONObject(map).toString() + ) + } +} + + +class TestRestHandler : RouterNanoHTTPD.GeneralHandler() { + private lateinit var looperHandler: Handler + + override fun get( + uriResource: RouterNanoHTTPD.UriResource, + urlParams: Map, + session: NanoHTTPD.IHTTPSession + ): NanoHTTPD.Response { + + val sessingParam = session.parameters!! + + Log.d(LOG_TAG_DEBUG, "GET !!") + + Log.d(LOG_TAG_DEBUG, sessingParam.toString()) + sessingParam.forEach { + Log.d(LOG_TAG_DEBUG, "${it.key} : ${it.value}") + } + + return NanoHTTPD.newFixedLengthResponse("Requested: \n$sessingParam\n") + } + + override fun post( + uriResource: RouterNanoHTTPD.UriResource?, + urlParams: MutableMap?, + session: NanoHTTPD.IHTTPSession? + ): NanoHTTPD.Response { + Log.d(LOG_TAG_DEBUG, "${this::class.simpleName} POST !!") + +// // header 확인 +// session?.headers?.forEach { +// Log.d(LOG_TAG_DEBUG, "${it.key}: ${it.value}") +// } + + if(!::looperHandler.isInitialized) + looperHandler = uriResource?.initParameter(Handler::class.java)?: Handler(Looper.getMainLooper()) + + val hashMap = HashMap() + session?.parseBody(hashMap) + + val receivedBody = if(hashMap.isNotEmpty()) hashMap["postData"] else session?.queryParameterString + if (receivedBody != null) { + val data = Json.decodeFromString(receivedBody) + + val json = Json.parseToJsonElement(receivedBody) + json.jsonObject.toMap().forEach { + Log.d(LOG_TAG_DEBUG, "receivedBody:: ${it.key} : ${it.value}") + } + +// if (session?.headers?.get("content-type")?.contains("application/json", true) == true) { +// Log.d(LOG_TAG_DEBUG, "BODY::\n${JSONObject(receivedBody).toString(4)}") +// } + + runCommand(data) + } + + return NanoHTTPD.newFixedLengthResponse("Received body:\n$receivedBody\n") + } + + private fun runCommand(data: RestTestRequest) { + when(data.cmdCode) { + 999 -> { + looperHandler.sendEmptyMessage(999) + } + in 100..199 -> { + looperHandler.sendEmptyMessage(data.cmdCode) + } + else -> { + Log.d(LOG_TAG_DEBUG, "Unknown command: $data") + } + } + } +} diff --git a/app/src/main/java/com/a2d2/mobilitygateway/StartActivity.kt b/app/src/main/java/com/a2d2/mobilitygateway/StartActivity.kt new file mode 100644 index 0000000..e15c262 --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/StartActivity.kt @@ -0,0 +1,167 @@ +package com.a2d2.mobilitygateway + +import android.content.Intent +import android.content.pm.PackageManager +import android.os.Build +import android.os.Bundle +import android.util.Log +import android.view.View +import android.view.WindowInsets +import android.view.WindowInsetsController +import android.widget.Toast +import androidx.activity.addCallback +import androidx.appcompat.app.AppCompatActivity +import androidx.core.app.ActivityCompat +import androidx.core.content.ContextCompat +import com.a2d2.mobilitygateway.databinding.ActivityMainBinding + + +@androidx.media3.common.util.UnstableApi // FERMAT(HSJ100): LOCAL MODULE +class StartActivity : AppCompatActivity() { + private val binding: ActivityMainBinding by lazy { + ActivityMainBinding.inflate(layoutInflater) + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(binding.root) + + Log.d(LOG_TAG_DEBUG, "----- START ${javaClass.simpleName} -----") + + setFullScreen() + + binding.modelToggle.setOnCheckedChangeListener{ _, isChecked -> + OD_MODEL = if (isChecked){ + ODModelFlavor.RTMDET + }else{ + ODModelFlavor.YOLOV8 + } + } + + binding.camToggle.setOnCheckedChangeListener{ _, isChecked -> + use_camera = isChecked + } + + binding.odToggle.setOnCheckedChangeListener{ _, isChecked -> + USE_OD_MODEL = isChecked + } + + binding.poseToggle.setOnCheckedChangeListener{ _, isChecked -> + USE_POSE_MODEL = isChecked + } + + binding.btnStartMG.setOnClickListener { + if (allPermissionsGranted()) { + startMobilityGateway() + } else { + ActivityCompat.requestPermissions( + this, REQUIRED_PERMISSIONS, PERMISSION_REQUEST_CODE + ) + } + } + + binding.btnExit.setOnClickListener { + finishAffinity() + } + onBackPressedDispatcher.addCallback { + finishAffinity() + } + + binding.txtRestIpAddr.text = RESTManager.ipAddr + binding.edtRestPort.hint = DEFAULT_REST_PORT.toString() + + binding.edtMqttIpAddr.hint = DEFAULT_MQTT_BROKER_IP + binding.edtMqttPort.hint = DEFAULT_MQTT_PORT.toString() + + binding.edtMqttUser.setText(MQTT_LOGIN_USERNAME) + binding.edtMqttPass.setText(MQTT_LOGIN_PASSWORD) + } + + private fun startMobilityGateway() { + val engineIntent = Intent(this, EngineActivity::class.java) + val restPort = binding.edtRestPort.text.toString().trim() + val mqttPort = binding.edtMqttPort.text.toString().trim() + val mqttIpAddr = binding.edtMqttIpAddr.text.toString().trim() + val mqttUser = binding.edtMqttUser.text.toString().trim() + val mqttPass = binding.edtMqttPass.text.toString().trim() + + engineIntent.putExtra("restPort", if(restPort.isEmpty()) DEFAULT_REST_PORT else restPort.toInt()) + engineIntent.putExtra("mqttPort", if(mqttPort.isEmpty()) DEFAULT_MQTT_PORT else mqttPort.toInt()) + engineIntent.putExtra("mqttIpAddr", mqttIpAddr.ifEmpty { DEFAULT_MQTT_BROKER_IP }) + engineIntent.putExtra("mqttUser", mqttUser) + engineIntent.putExtra("mqttPass", mqttPass) + + startActivity(engineIntent) + } + + private fun stopMobilityGateway() { + + } + + private fun allPermissionsGranted() = REQUIRED_PERMISSIONS.all { + ContextCompat.checkSelfPermission(baseContext, it) == PackageManager.PERMISSION_GRANTED + } + + override fun onRequestPermissionsResult( + requestCode: Int, + permissions: Array, + grantResults: IntArray + ) { + if (requestCode == PERMISSION_REQUEST_CODE) { + when { + allPermissionsGranted() -> { + startMobilityGateway() + } + else -> { + Toast.makeText(this, + "Permissions not granted".uppercase(), + Toast.LENGTH_SHORT).show() + finishAffinity() + } + } + } + super.onRequestPermissionsResult(requestCode, permissions, grantResults) + } + + private fun setFullScreen() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + window.setDecorFitsSystemWindows(false) + window.insetsController?.hide(WindowInsets.Type.systemBars() or WindowInsets.Type.navigationBars()) + window.insetsController?.systemBarsBehavior = WindowInsetsController.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE + } + else { + window.decorView.systemUiVisibility = + (View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY or + View.SYSTEM_UI_FLAG_FULLSCREEN or + View.SYSTEM_UI_FLAG_HIDE_NAVIGATION or + View.SYSTEM_UI_FLAG_LAYOUT_STABLE or + View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN or + View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION) + } + } + + override fun onWindowFocusChanged(hasFocus: Boolean) { + if(!hasFocus) return + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + window.setDecorFitsSystemWindows(false) + window.insetsController?.hide(WindowInsets.Type.systemBars() or WindowInsets.Type.navigationBars()) + window.insetsController?.systemBarsBehavior = WindowInsetsController.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE + } + else { + window.decorView.systemUiVisibility = + (View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY or + View.SYSTEM_UI_FLAG_FULLSCREEN or + View.SYSTEM_UI_FLAG_HIDE_NAVIGATION or + View.SYSTEM_UI_FLAG_LAYOUT_STABLE or + View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN or + View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION) + } + } + + override fun onDestroy() { + Log.d(LOG_TAG_DEBUG, "----- END ${javaClass.simpleName} -----") + + super.onDestroy() + } +} \ No newline at end of file diff --git a/app/src/main/java/com/a2d2/mobilitygateway/Util.kt b/app/src/main/java/com/a2d2/mobilitygateway/Util.kt new file mode 100644 index 0000000..6917003 --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/Util.kt @@ -0,0 +1,184 @@ +package com.a2d2.mobilitygateway + +import android.content.ContentValues +import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.Matrix +import android.graphics.Rect +import android.provider.MediaStore +import android.util.Base64 +import androidx.camera.core.ImageProxy +import androidx.core.graphics.scale +import org.json.JSONArray +import org.json.JSONException +import org.json.JSONObject +import java.io.ByteArrayOutputStream +import java.io.DataOutputStream +import java.io.FileOutputStream +import kotlin.math.max + + +class Util { + // string이 json format인지 확인 + fun isJson(json: String): Boolean { + try { + JSONObject(json) + } + catch (e: JSONException) { + try { + JSONArray(json) + } + catch (ne: JSONException) { + return false + } + } + return true + } + + fun isNotJson(json: String) = !isJson(json) + + // MQTT publish를 위한 base64 encoding + fun encodeToBase64(imageProxy: ImageProxy?, forceScale: Boolean = false): String? { + if (imageProxy == null) + return null + + return encodeToBase64(imageProxy.toBitmap(), forceScale) + } + + fun encodeToBase64(imageBitmap: Bitmap?, forceScale: Boolean = false): String? { + if (imageBitmap == null) + return null + + val stream = ByteArrayOutputStream() + + if (forceScale) { + val scaledBitmap = imageBitmap.scale(CAMERA_IMAGE_WIDTH, CAMERA_IMAGE_HEIGHT) + scaledBitmap.compress(Bitmap.CompressFormat.JPEG, 70, stream) + } + else { + imageBitmap.compress(Bitmap.CompressFormat.JPEG, 70, stream) + } + + return Base64.encodeToString(stream.toByteArray(), Base64.NO_WRAP) + } + + fun decodeFromBase64(base64: String): Bitmap { + val byteArray = Base64.decode(base64, Base64.NO_WRAP) + + return BitmapFactory.decodeByteArray(byteArray, 0, byteArray.size) + } + + // yolo type 좌표를 통해 bitmap image crop + fun cropBitmapUsingBbox(bitmap: Bitmap, bbox: ArrayList): Bitmap { + val w = bbox[2] * bitmap.width + val h = bbox[3] * bitmap.height + + val x = max(0f, (bbox[0] * bitmap.width) - (w / 2)) + val y = max(0f, (bbox[1] * bitmap.height) - (h / 2)) + + return Bitmap.createBitmap(bitmap, x.toInt(), y.toInt(), w.toInt(), h.toInt()) + } + + fun cropBitmapUsingRect(source: Bitmap, rect: Rect): Bitmap { + var width = rect.width() + var height = rect.height() + + if (rect.left < 0) rect.left = 0 + if (rect.top < 0) rect.top = 0 + + if ((rect.left + width) > source.width) { + width = source.width - rect.left + } + if ((rect.top + height) > source.height) { + height = source.height - rect.top + } + // Uncomment the below line if you want to save the input image. + // BitmapUtils.saveBitmap( context , croppedBitmap , "source" ) + return Bitmap.createBitmap(source, rect.left, rect.top, width, height) + } + + fun getRectFromBbox(bitmap: Bitmap, bbox: ArrayList): Rect { + val w = bbox[2] * bitmap.width + val h = bbox[3] * bitmap.height + + val x1 = max(0f, (bbox[0] * bitmap.width) - (w / 2)) + val y1 = max(0f, (bbox[1] * bitmap.height) - (h / 2)) + val x2 = x1 + w + val y2 = y1 + h + + return Rect(x1.toInt(), y1.toInt(), x2.toInt(), y2.toInt()) + } + + // Rotate the given `source` by `degrees`. + // See this SO answer -> https://stackoverflow.com/a/16219591/10878733 + fun rotateBitmap(source: Bitmap, degrees: Float ): Bitmap { + val matrix = Matrix() + + matrix.postRotate( degrees ) + + return Bitmap.createBitmap(source, 0, 0, source.width, source.height, matrix , false ) + } + + // 임의 길이의 string 생성 + fun getRandomString(length: Int) : String { + val charset = ('a'..'z') + ('A'..'Z') + ('0'..'9') + return (1..length) + .map { charset.random() } + .joinToString("") + } + + // map을 json 형태로 저장 + // context 필요 + fun mapToFile(context: Context, filename: String, message: HashMap) { + try { + // 저장위치: /data/data/com.a2d2.mobilitygateway/files + val outputStream = context.openFileOutput(filename, Context.MODE_PRIVATE) + val dos = DataOutputStream(outputStream) + + dos.write(JSONObject(message).toString().toByteArray()) + dos.flush() + dos.close() + outputStream.close() + } + catch (e: Exception) { + throw Exception("mapToFile fail:: $e") + } + } + + // bitmap image를 폰에 저장 + // context 필요 + fun bitmapToFile(context: Context, bitmap: Bitmap, filename: String) { + val contentValues = ContentValues().apply { + put(MediaStore.Images.Media.RELATIVE_PATH, "Pictures/AndroidMG") + put(MediaStore.MediaColumns.DISPLAY_NAME, filename) + put(MediaStore.MediaColumns.MIME_TYPE, "image/jpeg") + //put(MediaStore.Images.Media.IS_PENDING, 1) + } + + try { + val contentResolver = context.contentResolver + val uri = contentResolver.insert(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, contentValues) + + if(uri != null) { + val image = contentResolver.openFileDescriptor(uri, "w", null) + + if(image != null) { + val fos = FileOutputStream(image.fileDescriptor) + bitmap.compress(Bitmap.CompressFormat.JPEG, 100, fos) + //비트맵을 FileOutputStream를 통해 compress한다. + fos.close() + +// contentValues.clear() +// contentValues.put(MediaStore.Images.Media.IS_PENDING, 0) // 저장소 독점을 해제한다. +// contentResolver.update(uri, contentValues, null, null) + + image.close() + } + } + } + catch (e: Exception) { + throw Exception("bitmapToFile fail:: $e") + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/a2d2/mobilitygateway/biDetect/WatchDiscrimination.kt b/app/src/main/java/com/a2d2/mobilitygateway/biDetect/WatchDiscrimination.kt new file mode 100644 index 0000000..5d35647 --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/biDetect/WatchDiscrimination.kt @@ -0,0 +1,195 @@ +package com.a2d2.mobilitygateway.biDetect +import com.a2d2.mobilitygateway.MQTTResponseBISensorBaseInfo +import com.a2d2.mobilitygateway.MQTTResponseBISensorBaseOutputInfo +import com.a2d2.mobilitygateway.MQTTResponseBISensorWatchInfo +import org.json.JSONObject +import kotlin.math.* + + +class WatchDiscrimination(private var watchData : JSONObject) { + + private val statusList = arrayOf("workable","unworkable") + private var workerId: Any? = watchData["worker_id"] + private var deviceId: Any? = watchData["device_id"] + private var heartRate: Any? = watchData["heart_rate"] + private var boc: Any? = watchData["boc"] + private var bloodPressure: Any? = watchData["blood_pressure"] + private var bodyTemperature: Any? = watchData["body_temperature"] + private var stress: Any? = watchData["stress"] + + fun setInfo(watchData : JSONObject){ + workerId = watchData["worker_id"] + deviceId = watchData["device_id"] + heartRate = watchData["heart_rate"] + boc = watchData["boc"] + bloodPressure = watchData["blood_pressure"] + bodyTemperature = watchData["body_temperature"] + stress = watchData["stress"] + } + + fun getMessage():MQTTResponseBISensorWatchInfo? { + val returnMessage = MQTTResponseBISensorWatchInfo( + worker_id = if (watchData["worker_id"] != null) workerId.toString() else null, + device_id = if (watchData["device_id"] != null) deviceId.toString() else null, + heart_rate = discriminationHeartRate(), + boc = discriminationBoc(), + blood_pressure = discriminationBloodPressure(), + body_temperature = discriminationBodyTemperature(), + stress = discriminationStress() + ) + return if (detectCheck(returnMessage)) returnMessage else null + } + + private fun detectCheck(message:MQTTResponseBISensorWatchInfo):Boolean{ + var flag = false + if (message.heart_rate != null){ + if (message.heart_rate.output.status_idx == 1){ + flag = true + } + } + if (message.boc != null){ + if (message.boc.output.status_idx == 1){ + flag = true + } + } + if (message.blood_pressure != null){ + if (message.blood_pressure.output.status_idx == 1){ + flag = true + } + } + if (message.body_temperature != null){ + if (message.body_temperature.output.status_idx == 1){ + flag = true + } + } + if (message.stress != null){ + if (message.stress.output.status_idx == 1){ + flag = true + } + } + return flag + } + + private fun discriminationHeartRate(): MQTTResponseBISensorBaseInfo? { + //심박수 판별 함수 + val min = 60 + val max = 100 + var result: MQTTResponseBISensorBaseInfo? = null + var index = 0 + + if (heartRate == null) { + return null + } + else { + val parsingInput = heartRate.toString().toInt() + + index = if ((min <= parsingInput) and (parsingInput <= max)) 0 else 1 + + result = MQTTResponseBISensorBaseInfo( + input = parsingInput.toString(), + output = MQTTResponseBISensorBaseOutputInfo( + status_idx = index, + status_list = statusList + ) + ) + return result + } + } + + private fun discriminationBoc(): MQTTResponseBISensorBaseInfo? { + //혈중산소농도 + val threshold = 95 + var result: MQTTResponseBISensorBaseInfo? = null + var index = 0 + + if (boc == null) { + return null + } else { + val parsingInput = boc.toString().toInt() + index = if (parsingInput >= threshold) 0 else 1 + + result = MQTTResponseBISensorBaseInfo( + input = parsingInput.toString(), + output = MQTTResponseBISensorBaseOutputInfo( + status_idx = index, + status_list = statusList + ) + ) + return result + } + } + + private fun discriminationBloodPressure(): MQTTResponseBISensorBaseInfo? { + //혈압 + val thresholdSystolic = 120 // 수축기 + val thresholdDiastolic = 80 // 이완기 + var result: MQTTResponseBISensorBaseInfo? = null + var index = 0 + + if (bloodPressure == null) { + return null + } else {3 + val parsingList = bloodPressure.toString().split(",") + val systolic = max(parsingList[0].toInt() , parsingList[1].toInt()) + val diastolic = min(parsingList[0].toInt() , parsingList[1].toInt()) + + index = if ((systolic < thresholdSystolic) and (diastolic < thresholdDiastolic)) 0 else 1 + + result = MQTTResponseBISensorBaseInfo( + input = bloodPressure.toString(), + output = MQTTResponseBISensorBaseOutputInfo( + status_idx = index, + status_list = statusList + ) + ) + return result + } + } + + private fun discriminationBodyTemperature(): MQTTResponseBISensorBaseInfo? { + //체온 + val threshold = 37.5 + var result: MQTTResponseBISensorBaseInfo? = null + var index = 0 + + if (bodyTemperature == null) { + return null + } else { + val parsingInput = bodyTemperature.toString().toFloat() + + index = if (parsingInput < threshold) 0 else 1 + + result = MQTTResponseBISensorBaseInfo( + input = bodyTemperature.toString(), + output = MQTTResponseBISensorBaseOutputInfo( + status_idx = index, + status_list = statusList + ) + ) + return result + } + } + + private fun discriminationStress(): MQTTResponseBISensorBaseInfo?{ + var result: MQTTResponseBISensorBaseInfo? = null + var index = 0 + + if (stress == null){ + return null + } else{ + val parsingInput = stress.toString().toBoolean() + + index = if (!parsingInput) 0 else 1 + + result = MQTTResponseBISensorBaseInfo( + input = stress.toString(), + output = MQTTResponseBISensorBaseOutputInfo( + status_idx = index, + status_list = statusList + ) + ) + return result + } + } + +} \ No newline at end of file diff --git a/app/src/main/java/com/a2d2/mobilitygateway/faceRecognize/FRRectView.kt b/app/src/main/java/com/a2d2/mobilitygateway/faceRecognize/FRRectView.kt new file mode 100644 index 0000000..662259e --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/faceRecognize/FRRectView.kt @@ -0,0 +1,88 @@ +package com.a2d2.mobilitygateway.faceRecognize + +import android.content.Context +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Matrix +import android.graphics.Paint +import android.util.AttributeSet +import android.view.SurfaceHolder +import android.view.SurfaceView +import androidx.camera.core.CameraSelector +import androidx.core.graphics.toRectF +import com.a2d2.mobilitygateway.FRResult + +// Defines an overlay on which the boxes and text will be drawn. +class FRRectView(context: Context, attributeSet: AttributeSet) + : SurfaceView(context, attributeSet), SurfaceHolder.Callback { + + // Variables used to compute output2overlay transformation matrix + // These are assigned in FrameAnalyser.kt + var areDimsInit = false + var frameHeight = 0 + var frameWidth = 0 + + var cameraFacing : Int = CameraSelector.LENS_FACING_BACK + + // This var is assigned in FrameAnalyser.kt + var faceBoundingBoxes: ArrayList? = null + + private var output2OverlayTransform: Matrix = Matrix() + + // Paint for boxes and text + private val boxPaint = Paint().apply { + color = Color.parseColor("#4D90caf9") + style = Paint.Style.FILL + } + private val textPaint = Paint().apply { + strokeWidth = 2.0f + textSize = 32f + color = Color.WHITE + } + + + override fun surfaceCreated(holder: SurfaceHolder) { + TODO("Not yet implemented") + } + + + override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) { + TODO("Not yet implemented") + } + + + override fun surfaceDestroyed(holder: SurfaceHolder) { + TODO("Not yet implemented") + } + + + override fun onDraw(canvas: Canvas) { + if (faceBoundingBoxes != null) { + if (!areDimsInit) { + val viewWidth = width.toFloat() + val viewHeight = height.toFloat() + val xFactor: Float = viewWidth / frameWidth.toFloat() + val yFactor: Float = viewHeight / frameHeight.toFloat() + // Scale and mirror the coordinates ( required for front lens ) + output2OverlayTransform.preScale(xFactor, yFactor) + if( cameraFacing == CameraSelector.LENS_FACING_FRONT ) { + output2OverlayTransform.postScale(-1f, 1f, viewWidth / 2f, viewHeight / 2f) + } + areDimsInit = true + } + else { + for (face in faceBoundingBoxes!!) { + val boundingBox = face.bbox.toRectF() + output2OverlayTransform.mapRect(boundingBox) + canvas.drawRoundRect(boundingBox, 16f, 16f, boxPaint) + canvas.drawText( + face.label, + boundingBox.centerX(), + boundingBox.centerY(), + textPaint + ) + } + } + } + } +} diff --git a/app/src/main/java/com/a2d2/mobilitygateway/faceRecognize/FaceModels.kt b/app/src/main/java/com/a2d2/mobilitygateway/faceRecognize/FaceModels.kt new file mode 100644 index 0000000..fb3e0c6 --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/faceRecognize/FaceModels.kt @@ -0,0 +1,43 @@ +package com.a2d2.mobilitygateway.faceRecognize + + +data class FaceModelInfo( + val name: String, val assetsFilename: String, + val cosineThreshold: Float, val l2Threshold: Float, + val outputDims: Int, val inputDims: Int) + +class FaceModels { + companion object { + val FACENET = FaceModelInfo( + "FaceNet", "facenet.tflite", + 0.4f, + 10f, + 128, + 160 + ) + + val FACENET_512 = FaceModelInfo( + "FaceNet-512", "facenet_512.tflite", + 0.3f, + 23.56f, + 512, + 160 + ) + + val FACENET_QUANTIZED = FaceModelInfo( + "FaceNet Quantized", "facenet_int_quantized.tflite", + 0.4f, + 10f, + 128, + 160 + ) + + val FACENET_512_QUANTIZED = FaceModelInfo( + "FaceNet-512 Quantized", "facenet_512_int_quantized.tflite", + 0.3f, + 23.56f, + 512, + 160 + ) + } +} \ No newline at end of file diff --git a/app/src/main/java/com/a2d2/mobilitygateway/faceRecognize/FaceNetModel.kt b/app/src/main/java/com/a2d2/mobilitygateway/faceRecognize/FaceNetModel.kt new file mode 100644 index 0000000..bd7efda --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/faceRecognize/FaceNetModel.kt @@ -0,0 +1,98 @@ +package com.a2d2.mobilitygateway.faceRecognize + +import android.content.Context +import android.graphics.Bitmap +import android.util.Log +import com.a2d2.mobilitygateway.LOG_TAG_DEBUG +import org.tensorflow.lite.DataType +import org.tensorflow.lite.Interpreter +import org.tensorflow.lite.gpu.CompatibilityList +import org.tensorflow.lite.gpu.GpuDelegate +import org.tensorflow.lite.support.common.FileUtil +import org.tensorflow.lite.support.common.TensorOperator +import org.tensorflow.lite.support.image.ImageProcessor +import org.tensorflow.lite.support.image.TensorImage +import org.tensorflow.lite.support.image.ops.ResizeOp +import org.tensorflow.lite.support.tensorbuffer.TensorBuffer +import org.tensorflow.lite.support.tensorbuffer.TensorBufferFloat +import java.nio.ByteBuffer +import kotlin.math.max +import kotlin.math.pow +import kotlin.math.sqrt + + +class FaceNetModel(context : Context, + var model: FaceModelInfo = FaceModels.FACENET, + useGpu: Boolean = true, useXNNPack: Boolean = true) +{ + // Input image size for FaceNet model. + private val imgSize = model.inputDims + + // Output embedding size + val embeddingDim = model.outputDims + + private var interpreter : Interpreter + private val imageTensorProcessor = ImageProcessor.Builder() + .add( ResizeOp( imgSize , imgSize , ResizeOp.ResizeMethod.BILINEAR ) ) + .add( StandardizeOp() ) + .build() + + init { + // Initialize TFLiteInterpreter + val interpreterOptions = Interpreter.Options().apply { + // Add the GPU Delegate if supported. + // See -> https://www.tensorflow.org/lite/performance/gpu#android + if ( useGpu ) { + if ( CompatibilityList().isDelegateSupportedOnThisDevice ) { + addDelegate( GpuDelegate( CompatibilityList().bestOptionsForThisDevice )) + } + } + else { + // Number of threads for computation + numThreads = 4 + } + useXNNPACK = useXNNPack + useNNAPI = true + } + interpreter = Interpreter(FileUtil.loadMappedFile(context, model.assetsFilename ) , interpreterOptions ) + Log.d(LOG_TAG_DEBUG, "Using ${model.name} model.") + } + + // Gets an face embedding using FaceNet. + fun getFaceEmbedding( image : Bitmap ) : FloatArray { + return runFaceNet( convertBitmapToBuffer( image ))[0] + } + + // Run the FaceNet model. + private fun runFaceNet(inputs: Any): Array { + val t1 = System.currentTimeMillis() + val faceNetModelOutputs = Array(1){ FloatArray( embeddingDim ) } + + interpreter.run( inputs, faceNetModelOutputs ) + Log.d(LOG_TAG_DEBUG, "${model.name} Inference Speed in ms : ${System.currentTimeMillis() - t1}") + + return faceNetModelOutputs + } + + // Resize the given bitmap and convert it to a ByteBuffer + private fun convertBitmapToBuffer( image : Bitmap) : ByteBuffer { + return imageTensorProcessor.process( TensorImage.fromBitmap( image ) ).buffer + } + + // Op to perform standardization + // x' = ( x - mean ) / std_dev + class StandardizeOp : TensorOperator { + override fun apply(p0: TensorBuffer?): TensorBuffer { + val pixels = p0!!.floatArray + val mean = pixels.average().toFloat() + var std = sqrt( pixels.map{ pi -> ( pi - mean ).pow( 2 ) }.sum() / pixels.size.toFloat() ) + std = max( std , 1f / sqrt( pixels.size.toFloat() )) + for ( i in pixels.indices ) { + pixels[ i ] = ( pixels[ i ] - mean ) / std + } + val output = TensorBufferFloat.createFixedSize( p0.shape , DataType.FLOAT32 ) + output.loadArray( pixels ) + return output + } + } +} diff --git a/app/src/main/java/com/a2d2/mobilitygateway/faceRecognize/FrameAnalyser.kt b/app/src/main/java/com/a2d2/mobilitygateway/faceRecognize/FrameAnalyser.kt new file mode 100644 index 0000000..2ae5152 --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/faceRecognize/FrameAnalyser.kt @@ -0,0 +1,200 @@ +package com.a2d2.mobilitygateway.faceRecognize + +import android.annotation.SuppressLint +import android.graphics.Bitmap +import android.util.Log +import androidx.camera.core.ImageAnalysis +import androidx.camera.core.ImageProxy +import com.a2d2.mobilitygateway.FRResult +import com.a2d2.mobilitygateway.LOG_TAG_DEBUG +import com.a2d2.mobilitygateway.Util +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.face.Face +import com.google.mlkit.vision.face.FaceDetection +import com.google.mlkit.vision.face.FaceDetectorOptions +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlin.math.pow +import kotlin.math.sqrt + +// Analyser class to process frames and produce detections. +class FrameAnalyser( + private var rectView: FRRectView, + private var model: FaceNetModel +) : ImageAnalysis.Analyzer { + private val util: Util = Util() + + private val realTimeOpts = FaceDetectorOptions.Builder() + .setPerformanceMode( FaceDetectorOptions.PERFORMANCE_MODE_FAST ) + .build() + private val detector = FaceDetection.getClient(realTimeOpts) + + private val nameScoreHashmap = HashMap>() + private var subject = FloatArray( model.embeddingDim ) + + // Used to determine whether the incoming frame should be dropped or processed. + private var isProcessing = false + + // Store the face embeddings in a ( String , FloatArray ) ArrayList. + // Where String -> name of the person and FloatArray -> Embedding of the face. + var faceList = ArrayList>() + + private var t1 : Long = 0L + + // <-------------- User controls ---------------------------> + + // Use any one of the two metrics, "cosine" or "l2" + private val metricToBeUsed = "l2" + + // <--------------------------------------------------------> + + + @SuppressLint("UnsafeOptInUsageError") + override fun analyze(image: ImageProxy) { + // If the previous frame is still being processed, then skip this frame + //if ( isProcessing || faceList.size == 0 ) { // TODO:: 이게 원본. 비교얼굴이 있는지 확인 + if ( isProcessing ) { + image.close() + return + } + else { + isProcessing = true + + // Rotated bitmap for the FaceNet model + val cameraXImage = image.image!! + var frameBitmap = Bitmap.createBitmap( cameraXImage.width , cameraXImage.height , Bitmap.Config.ARGB_8888 ) + // TODO:: var frameBitmap = image.toBitmap() + + frameBitmap.copyPixelsFromBuffer( image.planes[0].buffer ) + frameBitmap = util.rotateBitmap(frameBitmap, image.imageInfo.rotationDegrees.toFloat()) + //val frameBitmap = BitmapUtils.imageToBitmap( image.image!! , image.imageInfo.rotationDegrees ) + + // Configure frameHeight and frameWidth for output2overlay transformation matrix. + if ( !rectView.areDimsInit ) { + rectView.frameHeight = frameBitmap.height + rectView.frameWidth = frameBitmap.width + } + + val inputImage = InputImage.fromBitmap( frameBitmap , 0 ) + detector.process(inputImage) + .addOnSuccessListener { faces -> + CoroutineScope( Dispatchers.Default ).launch { + runModel( faces , frameBitmap ) + } + } + .addOnCompleteListener { + image.close() + } + } + } + + private suspend fun runModel( faces : List , cameraFrameBitmap : Bitmap ){ + withContext( Dispatchers.Default ) { + t1 = System.currentTimeMillis() + val predictions = ArrayList() + for (face in faces) { + try { + // Crop the frame using face.boundingBox. + // Convert the cropped Bitmap to a ByteBuffer. + // Finally, feed the ByteBuffer to the FaceNet model. + val croppedBitmap = util.cropBitmapUsingRect( cameraFrameBitmap , face.boundingBox ) + subject = model.getFaceEmbedding( croppedBitmap ) + + // Perform clustering ( grouping ) + // Store the clusters in a HashMap. Here, the key would represent the 'name' + // of that cluster and ArrayList would represent the collection of all + // L2 norms/ cosine distances. + for ( i in 0 until faceList.size ) { + // If this cluster ( i.e an ArrayList with a specific key ) does not exist, + // initialize a new one. + if ( nameScoreHashmap[ faceList[ i ].first ] == null ) { + // Compute the L2 norm and then append it to the ArrayList. + val p = ArrayList() + if ( metricToBeUsed == "cosine" ) { + p.add( cosineSimilarity( subject , faceList[ i ].second ) ) + } + else { + p.add( L2Norm( subject , faceList[ i ].second ) ) + } + nameScoreHashmap[ faceList[ i ].first ] = p + } + // If this cluster exists, append the L2 norm/cosine score to it. + else { + if ( metricToBeUsed == "cosine" ) { + nameScoreHashmap[ faceList[ i ].first ]?.add( cosineSimilarity( subject , faceList[ i ].second ) ) + } + else { + nameScoreHashmap[ faceList[ i ].first ]?.add( L2Norm( subject , faceList[ i ].second ) ) + } + } + } + + // Compute the average of all scores norms for each cluster. + val avgScores = nameScoreHashmap.values.map{ scores -> scores.toFloatArray().average() } + Log.d(LOG_TAG_DEBUG, "Average score for each user: $nameScoreHashmap" ) + + val names = nameScoreHashmap.keys.toTypedArray() + nameScoreHashmap.clear() + + var bestScoreUserName: String = "Unknown" + if (faceList.size > 0) { + // Calculate the minimum L2 distance from the stored average L2 norms. + bestScoreUserName = if ( metricToBeUsed == "cosine" ) { + // In case of cosine similarity, choose the highest value. + if ( avgScores.maxOrNull()!! > model.model.cosineThreshold ) { + names[ avgScores.indexOf( avgScores.maxOrNull()!! ) ] + } + else { + "Unknown" + } + } else { + // In case of L2 norm, choose the lowest value. + if ( avgScores.minOrNull()!! > model.model.l2Threshold ) { + "Unknown" + } + else { + names[ avgScores.indexOf( avgScores.minOrNull()!! ) ] + } + } + } + Log.d(LOG_TAG_DEBUG, "Person identified as $bestScoreUserName" ) + predictions.add( + FRResult( + face.boundingBox, + bestScoreUserName , + "" + ) + ) + } + catch ( e : Exception ) { + // If any exception occurs with this box and continue with the next boxes. + Log.e(LOG_TAG_DEBUG, "Exception in FrameAnalyser : ${e.message}" ) + continue + } + Log.d(LOG_TAG_DEBUG, "Inference time -> ${System.currentTimeMillis() - t1}") + } + withContext( Dispatchers.Main ) { + // Clear the FRRectView and set the new results ( boxes ) to be displayed. + rectView.faceBoundingBoxes = predictions + rectView.invalidate() + isProcessing = false + } + } + } + + // Compute the L2 norm of ( x2 - x1 ) + private fun L2Norm( x1 : FloatArray, x2 : FloatArray ) : Float { + return sqrt( x1.mapIndexed{ i , xi -> (xi - x2[ i ]).pow( 2 ) }.sum() ) + } + + // Compute the cosine of the angle between x1 and x2. + private fun cosineSimilarity( x1 : FloatArray , x2 : FloatArray ) : Float { + val mag1 = sqrt( x1.map { it * it }.sum() ) + val mag2 = sqrt( x2.map { it * it }.sum() ) + val dot = x1.mapIndexed{ i , xi -> xi * x2[ i ] }.sum() + return dot / (mag1 * mag2) + } + +} \ No newline at end of file diff --git a/app/src/main/java/com/a2d2/mobilitygateway/objectDetect/BitmapDraw.kt b/app/src/main/java/com/a2d2/mobilitygateway/objectDetect/BitmapDraw.kt new file mode 100644 index 0000000..fa458e3 --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/objectDetect/BitmapDraw.kt @@ -0,0 +1,319 @@ +package com.a2d2.mobilitygateway.objectDetect + +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import com.a2d2.mobilitygateway.ODModelFlavor +import com.a2d2.mobilitygateway.ODResult +import com.a2d2.mobilitygateway.OD_CLASSES +import com.a2d2.mobilitygateway.OD_MODEL +import com.a2d2.mobilitygateway.VIEW_CONFIDENCE_SCORE +import kotlin.math.round + +class BitmapDraw { + private var odResult: ArrayList? = null + private var classes: Array = OD_CLASSES + + private var poseResult: ArrayList? = null + + private var bitmapImg : Bitmap? = null + private lateinit var canvas: Canvas + + private val textPaint = Paint().also { + it.textSize = 30f + it.color = Color.WHITE + } + + private val pointPaint = Paint().apply { + color = Color.RED + style = Paint.Style.STROKE + strokeWidth = 10f + } + + private val linePaint = Paint().apply { + color = Color.YELLOW + style = Paint.Style.FILL + strokeWidth = 5f + } + + private fun setBitmap(bitmap: Bitmap){ + this.bitmapImg = bitmap + this.canvas = Canvas(this.bitmapImg!!) + } + + fun getBitmap(): Bitmap? { + return this.bitmapImg + } + + private fun setOd(results: ArrayList) { + if (this.bitmapImg != null){ + when(OD_MODEL){ + ODModelFlavor.YOLOV8 -> { // scale 구하기 + val scaleX = bitmapImg!!.width / OnnxManager.INPUT_SIZE.toFloat() + val scaleY = scaleX * (bitmapImg!!.height / bitmapImg!!.width.toFloat()) + val realY = bitmapImg!!.width * (bitmapImg!!.height / bitmapImg!!.width.toFloat()) + val diffY = realY - bitmapImg!!.height + + results.forEach { + it.rectBitmap.left *= scaleX + it.rectBitmap.right *= scaleX + it.rectBitmap.top = it.rectBitmap.top * scaleY - (diffY / 2f) + it.rectBitmap.bottom = it.rectBitmap.bottom * scaleY - (diffY / 2f) + }} + ODModelFlavor.RTMDET -> { + val scaleX = bitmapImg!!.width / OnnxManager.INPUT_SIZE.toFloat() + val scaleY = bitmapImg!!.height / OnnxManager.INPUT_SIZE.toFloat() + + results.forEach { + it.rectBitmap.left = it.rectBitmap.left * scaleX + it.rectBitmap.right = it.rectBitmap.right * scaleX + it.rectBitmap.top = it.rectBitmap.top * scaleY + it.rectBitmap.bottom = it.rectBitmap.bottom * scaleY + }} + } + this.odResult = results + } + } + + private fun setPose(list: ArrayList) { + this.poseResult = list + } + + fun onLabelDraw(bitmap: Bitmap, od:ArrayList, pose:ArrayList){ + this.setBitmap(bitmap) + this.setOd(od) + this.setPose(pose) + + if (pose != null){ + poseDraw() + } + if (od != null){ + odDraw() + } + } + + // * od functions + private fun odDraw() { + //그림 그리기 + odResult?.forEach { + var labelName = classes[it.classIndex] + if (VIEW_CONFIDENCE_SCORE){ + labelName = classes[it.classIndex] + ", " + round(it.score * 100) + "%" + } + this.canvas.drawRect(it.rectBitmap, findPaint(it.classIndex)) + this.canvas.drawText( + labelName, + it.rectBitmap.left + 10, + it.rectBitmap.top + 60, + textPaint + ) + } + } + + //paint 지정 + private fun findPaint(classIndex: Int): Paint { + val paint = Paint() + paint.style = Paint.Style.STROKE // 빈 사각형 그림 + paint.strokeWidth = 7.0f // 굵기 7 + paint.strokeCap = Paint.Cap.ROUND // 끝을 뭉특하게 + paint.strokeJoin = Paint.Join.ROUND // 끝 주위도 뭉특하게 + paint.strokeMiter = 100f // 뭉특한 정도는 100도 + + //임의로 지정한 색상 + paint.color = when (classIndex) { + 0, 45, 18, 19, 22, 30, 42, 43, 44, 61, 71, 72 -> Color.WHITE + 11, 12, 13, 14, 25, 37, 38, 79 -> Color.BLUE + 1, 3, 6, 8, 10, 32, 47, 49, 51, 52 -> Color.RED + 23, 46, 48 -> Color.YELLOW + 34, 35, 36, 54, 59, 60, 73, 77, 78 -> Color.GRAY + 24, 26, 27, 28, 62, 64, 65, 66, 67, 68, 69, 74, 75 -> Color.BLACK + 2, 4, 5, 7, 9, 29, 33, 39, 41, 58, 50 -> Color.GREEN + 15, 16, 17, 20, 21, 31, 40, 55, 57, 63 -> Color.DKGRAY + 70, 76 -> Color.LTGRAY + else -> Color.DKGRAY + } + return paint + } + + + // * pose functions + private fun poseDraw() { + //점, 선 그리기 + drawPointsAndLines() + } + + private fun drawPointsAndLines() { + if (this.bitmapImg != null) { + val scaleX = bitmapImg!!.width / OnnxManager.INPUT_SIZE.toFloat() + val scaleY = scaleX * 9f / 16f + val realY = bitmapImg!!.width * 9f / 16f + val diffY = realY - bitmapImg!!.height + + val kPointsThreshold = 0.35f + poseResult?.forEach { + val points = FloatArray(34) + for ((a, i) in (points.indices step 2).withIndex()) { + if (it[i + 7 + a] > kPointsThreshold) { + points[i] = it[i + 5 + a] * scaleX + points[i + 1] = it[i + 6 + a] * scaleY - (diffY / 2f) + } + } + drawPoint(points) + drawLines(points) + } + } + } + + private fun drawPoint(points: FloatArray) { + for (i in points.indices step 2) { + val xPos = points[i] + val yPos = points[i + 1] + if (xPos > 0 && yPos > 0) { + this.canvas.drawPoint(xPos, yPos, pointPaint) + } + } + } + + private fun drawLines(points: FloatArray) { + // 점과 점사이에 직선 그리기 + // keypoint 순서 + // 0번 == 코 + // 1번 == 오른쪽 눈 + // 2번 == 왼쪽 눈 + // 3번 == 오른쪽 귀 + // 4번 == 왼쪽 귀 + // 5번 == 오른쪽 어깨 + // 6번 == 왼쪽 어깨 + // 7번 == 오른쪽 팔꿈치 + // 8번 == 왼쪽 팔꿈치 + // 9번 == 오른쪽 손목 + // 10번 == 왼쪽 손목 + // 11번 == 오른쪽 골반 + // 12번 == 왼쪽 골반 + // 13번 == 오른쪽 무릎 + // 14번 == 왼쪽 무릎 + // 15번 == 오른쪽 발 + // 16번 == 왼쪽 발 + + // 코, 오른쪽 눈 연결 + var startX = points[0] + var startY = points[1] + var stopX = points[2] + var stopY = points[3] + drawLine(startX, startY, stopX, stopY) + // 코, 왼쪽 눈 연결 + startX = points[0] + startY = points[1] + stopX = points[4] + stopY = points[5] + drawLine(startX, startY, stopX, stopY) + //오른쪽 눈 귀 연결 + startX = points[2] + startY = points[3] + stopX = points[8] + stopY = points[9] + drawLine(startX, startY, stopX, stopY) + //왼쪽 눈 귀 연결 + startX = points[4] + startY = points[5] + stopX = points[8] + stopY = points[9] + drawLine(startX, startY, stopX, stopY) + //오른쪽 귀 어깨 연결 + startX = points[6] + startY = points[7] + stopX = points[10] + stopY = points[11] + drawLine(startX, startY, stopX, stopY) + //왼쪽 귀 어깨 연결 + startX = points[8] + startY = points[9] + stopX = points[12] + stopY = points[13] + drawLine(startX, startY, stopX, stopY) + //오른쪽 어깨 팔꿈치 연결 + startX = points[10] + startY = points[11] + stopX = points[14] + stopY = points[15] + drawLine(startX, startY, stopX, stopY) + //왼쪽 어깨 팔꿈치 연결 + startX = points[12] + startY = points[13] + stopX = points[16] + stopY = points[17] + drawLine(startX, startY, stopX, stopY) + //오른쪽 어깨 골반 연결 + startX = points[10] + startY = points[11] + stopX = points[22] + stopY = points[23] + drawLine(startX, startY, stopX, stopY) + //왼쪽 어깨 골반 연결 + startX = points[12] + startY = points[13] + stopX = points[24] + stopY = points[25] + drawLine(startX, startY, stopX, stopY) + //오른쪽 팔꿈치 손목 연결 + startX = points[14] + startY = points[15] + stopX = points[18] + stopY = points[19] + drawLine(startX, startY, stopX, stopY) + //왼쪽 팔꿈치 손목 연결 + startX = points[16] + startY = points[17] + stopX = points[20] + stopY = points[21] + drawLine(startX, startY, stopX, stopY) + //오른쪽 골반 무릎 연결 + startX = points[22] + startY = points[23] + stopX = points[26] + stopY = points[27] + drawLine(startX, startY, stopX, stopY) + //왼쪽 골반 무릎 연결 + startX = points[24] + startY = points[25] + stopX = points[28] + stopY = points[29] + drawLine(startX, startY, stopX, stopY) + //오른쪽 무릎 발 연결 + startX = points[26] + startY = points[27] + stopX = points[30] + stopY = points[31] + drawLine(startX, startY, stopX, stopY) + //왼쪽 무릎 발 연결 + startX = points[28] + startY = points[29] + stopX = points[32] + stopY = points[33] + drawLine(startX, startY, stopX, stopY) + //어깨 좌우 연결 + startX = points[10] + startY = points[11] + stopX = points[12] + stopY = points[13] + drawLine(startX, startY, stopX, stopY) + //골반 좌우 연결 + startX = points[22] + startY = points[23] + stopX = points[24] + stopY = points[25] + drawLine(startX, startY, stopX, stopY) + } + + private fun drawLine( + startX: Float, + startY: Float, + stopX: Float, + stopY: Float, + ) { + if (startX > 0 && startY > 0 && stopX > 0 && stopY > 0) { + this.canvas.drawLine(startX, startY, stopX, stopY, linePaint) + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/a2d2/mobilitygateway/objectDetect/OnnxManager.kt b/app/src/main/java/com/a2d2/mobilitygateway/objectDetect/OnnxManager.kt new file mode 100644 index 0000000..2602d0d --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/objectDetect/OnnxManager.kt @@ -0,0 +1,525 @@ +package com.a2d2.mobilitygateway.objectDetect + +import com.a2d2.mobilitygateway.* + +import ai.onnxruntime.OnnxTensor +import ai.onnxruntime.OrtEnvironment +import ai.onnxruntime.OrtSession +import android.content.Context +import android.graphics.Bitmap +import android.graphics.RectF +import java.io.BufferedReader +import java.io.File +import java.io.FileOutputStream +import java.io.InputStreamReader +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.FloatBuffer +import java.util.* +import kotlin.collections.ArrayList +import kotlin.math.max +import kotlin.math.min + +class OnnxManager(val context: Context, + val odFlavor: ODModelFlavor = ODModelFlavor.YOLOV8) { + lateinit var classes: Array + private lateinit var ortEnvironment: OrtEnvironment + private lateinit var session: OrtSession + private lateinit var ortEnvironmentPose: OrtEnvironment + private lateinit var sessionPose: OrtSession + + companion object { + const val BATCH_SIZE = 1 + const val INPUT_SIZE = TRAIN_IMAGE_SIZE + const val PIXEL_SIZE = 3 + } + + init { + loadModel() + loadLabel() + + // for OD + if (USE_OD_MODEL){ + ortEnvironment = OrtEnvironment.getEnvironment() + session = ortEnvironment.createSession(context.filesDir.absolutePath + "/" + FILENAME_OD_MODEL, + OrtSession.SessionOptions() + )} + + // for Pose + if (USE_POSE_MODEL){ + ortEnvironmentPose = OrtEnvironment.getEnvironment() + sessionPose = ortEnvironmentPose.createSession(context.filesDir.absolutePath + "/" + FILENAME_POSE_MODEL, + OrtSession.SessionOptions() + )} + } + + private fun bitmapToFloatBuffer(imageBitmap: Bitmap, inputSize: Int = INPUT_SIZE): FloatBuffer { + val imageSTD = 255.0f + + val bitmap = Bitmap.createScaledBitmap(imageBitmap, inputSize, inputSize, true) + + val area = inputSize * inputSize + val cap = area * BATCH_SIZE * PIXEL_SIZE + val order = ByteOrder.nativeOrder() + val buffer = ByteBuffer.allocateDirect(cap * Float.SIZE_BYTES).order(order).asFloatBuffer() + + val bitmapData = IntArray(area) //한 사진에서 대한 정보, 640x640 사이즈 + bitmap.getPixels( + bitmapData, + 0, + bitmap.width, + 0, + 0, + bitmap.width, + bitmap.height + ) // bitmapData 배열에 pixel 정보 담기 + + //배열에서 하나씩 가져와서 buffer 에 담기 + for (i in 0 until inputSize - 1) { + for (j in 0 until inputSize - 1) { + val idx = inputSize * i + j + val pixelValue = bitmapData[idx] + // 위에서 부터 차례대로 R 값 추출, G 값 추출, B값 추출 -> 255로 나누어서 0~1 사이로 정규화 + buffer.put(idx, ((pixelValue shr 16 and 0xff) / imageSTD)) + buffer.put(idx + area, ((pixelValue shr 8 and 0xff) / imageSTD)) + buffer.put(idx + area * 2, ((pixelValue and 0xff) / imageSTD)) + //원리 bitmap == ARGB 형태의 32bit, R값의 시작은 16bit (16 ~ 23bit 가 R영역), 따라서 16bit 를 쉬프트 + //그럼 A값이 사라진 RGB 값인 24bit 가 남는다. 이후 255와 AND 연산을 통해 맨 뒤 8bit 인 R값만 가져오고, 255로 나누어 정규화를 한다. + //다시 8bit 를 쉬프트 하여 R값을 제거한 G,B 값만 남은 곳에 다시 AND 연산, 255 정규화, 다시 반복해서 RGB 값을 buffer 에 담는다. + } + } + buffer.rewind() // position 0 + + return buffer + } + + private fun bitmapToFloatBufferRtmdet(imageBitmap: Bitmap, inputSize: Int = INPUT_SIZE): FloatBuffer { + val bitmap = Bitmap.createScaledBitmap(imageBitmap, inputSize, inputSize, true) + + val area = inputSize * inputSize + val cap = area * BATCH_SIZE * PIXEL_SIZE + val buffer = ByteBuffer + .allocateDirect(cap * Float.SIZE_BYTES) + .order(ByteOrder.nativeOrder()) + .asFloatBuffer() + + // mmdet default (to_rgb=True) + val meanR = 123.675f; val meanG = 116.28f; val meanB = 103.53f + val stdR = 58.395f; val stdG = 57.12f; val stdB = 57.375f + + val data = IntArray(area) + bitmap.getPixels(data, 0, bitmap.width, 0, 0, bitmap.width, bitmap.height) + + // CHW 채널 순서 + for (i in 0 until inputSize - 1) { + for (j in 0 until inputSize - 1) { + val idx = inputSize * i + j + val p = data[idx] + val r = ((p shr 16) and 0xff).toFloat() + val g = ((p shr 8) and 0xff).toFloat() + val b = (p and 0xff).toFloat() + + buffer.put(idx, (r - meanR) / stdR) + buffer.put(idx + area, (g - meanG) / stdG) + buffer.put(idx + area*2, (b - meanB) / stdB) + } + } + buffer.rewind() + return buffer + } + + private fun loadModel() { + // onnx 파일 불러오기 + val assetManager = context.assets + + if (USE_OD_MODEL){ + val outputFileOD = File(context.filesDir, FILENAME_OD_MODEL) + assetManager.open(FILENAME_OD_MODEL).use { inputStream -> + FileOutputStream(outputFileOD).use { outputStream -> + val buffer = ByteArray(4 * 1024) + var read: Int + while (inputStream.read(buffer).also { read = it } != -1) { + outputStream.write(buffer, 0, read) + } + } + } + } + + if (USE_POSE_MODEL){ + val outputFilePose = File(context.filesDir, FILENAME_POSE_MODEL) + assetManager.open(FILENAME_POSE_MODEL).use { inputStream -> + FileOutputStream(outputFilePose).use { outputStream -> + val buffer = ByteArray(4 * 1024) + var read: Int + while (inputStream.read(buffer).also { read = it } != -1) { + outputStream.write(buffer, 0, read) + } + } + } + } + } + + private fun loadLabel() { + // label txt 파일 불러오기 + BufferedReader(InputStreamReader(context.assets.open(FILENAME_OD_LABEL))).use { reader -> + var line: String? + val classList = ArrayList() + while (reader.readLine().also { line = it } != null) { + classList.add(line!!) + } + classes = classList.toTypedArray() + } + } + + private fun outputsToNPMSPredictions(outputs: Array<*>): ArrayList { + val confidenceThreshold = 0.40f + val results = ArrayList() + val rows: Int + val cols: Int + + (outputs[0] as Array<*>).also { + rows = it.size + cols = (it[0] as FloatArray).size + } + + //배열의 형태를 [84 8400] -> [8400 84] 로 변환 + val output = Array(cols) { FloatArray(rows) } + for (i in 0 until rows) { + for (j in 0 until cols) { + output[j][i] = ((((outputs[0]) as Array<*>)[i]) as FloatArray)[j] + } + } + + for (i in 0 until cols) { + var detectionClass: Int = -1 + var maxScore = 0f + val classArray = FloatArray(classes.size) + // label 만 따로 빼서 1차원 배열을 만든다.(0~3은 좌표값임) + System.arraycopy(output[i], 4, classArray, 0, classes.size) + // label 중에서 가장 큰 값을 선정한다. + for (j in classes.indices) { + if (classArray[j] > maxScore) { + detectionClass = j + maxScore = classArray[j] + } + } + + //만약 80개의 coco dataset 중 가장 큰 확률값이 특정값을 (현재는 40% 확률) 넘어서면 해당 값을 저장한다. + if (maxScore > confidenceThreshold) { + val xPos = output[i][0] + val yPos = output[i][1] + val width = output[i][2] + val height = output[i][3] + //사각형은 화면 밖으로 나갈 수 없으니 화면을 넘기면 최대 화면 값을 가지게 한다. + val rectF = RectF( + max(0f, xPos - width / 2f), + max(0f, yPos - height / 2f), + min(INPUT_SIZE - 1f, xPos + width / 2f), + min(INPUT_SIZE - 1f, yPos + height / 2f) + ) + val rectBitmap = RectF( + max(0f, xPos - width / 2f), + max(0f, yPos - height / 2f), + min(INPUT_SIZE - 1f, xPos + width / 2f), + min(INPUT_SIZE - 1f, yPos + height / 2f) + ) + val bbox = arrayListOf( + (xPos / INPUT_SIZE).toFloat(), + (yPos / INPUT_SIZE).toFloat(), + (width / INPUT_SIZE).toFloat(), + (height / INPUT_SIZE).toFloat() + ) + + val result = ODResult(detectionClass, maxScore, rectF, rectBitmap, bbox) + results.add(result) + } + } + return nms(results) + } + + private fun nms(results: ArrayList): ArrayList { + val list = ArrayList() + + for (i in classes.indices) { + //1.클래스 (라벨들) 중에서 가장 높은 확률값을 가졌던 클래스 찾기 + val pq = PriorityQueue(50) { o1, o2 -> + o1.score.compareTo(o2.score) + } + val classResults = results.filter { it.classIndex == i } + pq.addAll(classResults) + + //NMS 처리 + while (pq.isNotEmpty()) { + // 큐 안에 속한 최대 확률값을 가진 class 저장 + val detections = pq.toTypedArray() + val max = detections[0] + list.add(max) + pq.clear() + + // 교집합 비율 확인하고 40%넘기면 제거 + for (k in 1 until detections.size) { + val detection = detections[k] + val rectF = detection.rectF + val iouThresh = 0.4f + if (boxIOU(max.rectF, rectF) < iouThresh) { + pq.add(detection) + } + } + } + } + return list + } + + // NEW: RTMDet(mmdet) ONNX 후처리 + private fun outputsToRTMDetPredictions(ortResult: OrtSession.Result): ArrayList { + val confidenceThreshold = 0.40f + val results = ArrayList() + + // RTMDet( mmdeploy / mmdet2onnx )는 보통 + // dets: [1, N, 5] (x1,y1,x2,y2,score) + labels: [1, N] + // 혹은 dets: [N, 5] + labels: [N] + // 형태. 두 경우를 모두 처리. + val detsAny = ortResult.get(0).value + val labelsAny = if (ortResult.size() > 1) ortResult.get(1).value else null + + // 헬퍼: (x1,y1,x2,y2,score) 한 줄을 ODResult로 변환 + fun addOne(x1: Float, y1: Float, x2: Float, y2: Float, score: Float, labelIdx: Int) { + if (score < confidenceThreshold) return + + val w = x2 - x1 + val h = y2 - y1 + val cx = x1 + w / 2f + val cy = y1 + h / 2f + + val rectF = RectF( + max(0f, x1), max(0f, y1), + min(INPUT_SIZE - 1f, x2), min(INPUT_SIZE - 1f, y2) + ) + + val bbox = arrayListOf( + (cx / INPUT_SIZE), (cy / INPUT_SIZE), + (w / INPUT_SIZE), (h / INPUT_SIZE) + ) + + val rectBitmap = RectF(rectF) + + results.add(ODResult(labelIdx, score, rectF, rectBitmap, bbox)) + } + + // 케이스 A: dets [1, N, 5], labels [1, N] + if (detsAny is Array<*> && detsAny.size == 1 && detsAny[0] is Array<*>) { + val dets = (detsAny as Array>)[0] // [N,5] + val labels = when (labelsAny) { + is Array<*> -> { + // int64가 float로 올 수도, long[]/int[]로 올 수도 있음 + val l0 = labelsAny[0] + when (l0) { + is LongArray -> l0.map { it.toInt() }.toIntArray() + is IntArray -> l0 + is FloatArray -> l0.map { it.toInt() }.toIntArray() + else -> IntArray(dets.size) { 0 } + } + } + is LongArray -> labelsAny.map { it.toInt() }.toIntArray() + is IntArray -> labelsAny + is FloatArray -> labelsAny.map { it.toInt() }.toIntArray() + else -> IntArray(dets.size) { 0 } + } + + for (i in dets.indices) { + val d = dets[i] + // d: [x1,y1,x2,y2,score] + addOne(d[0], d[1], d[2], d[3], d[4], labels.getOrElse(i){0}) + } + + // 케이스 B: dets [N,5], labels [N] + } else if (detsAny is Array<*>) { + val dets = detsAny as Array // [N,5] + val labels = when (labelsAny) { + is LongArray -> labelsAny.map { it.toInt() }.toIntArray() + is IntArray -> labelsAny + is FloatArray -> labelsAny.map { it.toInt() }.toIntArray() + else -> IntArray(dets.size) { 0 } + } + + for (i in dets.indices) { + val d = dets[i] + addOne(d[0], d[1], d[2], d[3], d[4], labels.getOrElse(i){0}) + } + + } else { + // 다른 형태(후처리 미포함 ONNX 등)인 경우: 필요시 여기서 추가 분기 + // throw IllegalStateException("Unsupported RTMDet ONNX output shape") + return results // 빈 결과 + } + + // 기존 NMS 재사용 (클래스별로 NMS 수행) +// return nms(results) + return results + } + + private fun outputsToNPMSPredictionsPose(outputs: Array<*>): ArrayList { + val confidenceThreshold = 0.4f + val rows: Int + val cols: Int + val results = ArrayList() + + (outputs[0] as Array<*>).also { + rows = it.size + cols = (it[0] as FloatArray).size + } + + //배열 형태를 [56 8400] -> [8400 56] 으로 변환 + val output = Array(cols) { FloatArray(rows) } + for (i in 0 until rows) { + for (j in 0 until cols) { + output[j][i] = (((outputs[0] as Array<*>)[i]) as FloatArray)[j] + } + } + + for (i in 0 until cols) { + // 바운딩 박스의 특정 확률을 넘긴 경우에만 xywh -> xy xy 형태로 변환 후 nms 처리 + if (output[i][4] > confidenceThreshold) { + val xPos = output[i][0] + val yPos = output[i][1] + val width = output[i][2] + val height = output[i][3] + + val x1 = max(xPos - width / 2f, 0f) + val x2 = min(xPos + width / 2f, INPUT_SIZE - 1f) + val y1 = max(yPos - height / 2f, 0f) + val y2 = min(yPos + height / 2f, INPUT_SIZE - 1f) + + output[i][0] = x1 + output[i][1] = y1 + output[i][2] = x2 + output[i][3] = y2 + + results.add(output[i]) + } + } + return nmsPose(results) + } + + private fun nmsPose(results: ArrayList): ArrayList { + val list = ArrayList() + //results 안에 있는 conf 값 중에서 제일 높은 애를 기준으로 NMS 가 겹치는 애들을 제거 + val pq = PriorityQueue(5) { o1, o2 -> + o1[4].compareTo(o2[4]) + } + + pq.addAll(results) + + while (pq.isNotEmpty()) { + // 큐 안에 속한 최대 확률값을 가진 FloatArray 저장 + val detections = pq.toTypedArray() + val max = detections[0] + list.add(max) + pq.clear() + + // 교집합 비율 확인하고 50% 넘기면 제거 + for (k in 1 until detections.size) { + val detection = detections[k] + val rectF = RectF(detection[0], detection[1], detection[2], detection[3]) + val maxRectF = RectF(max[0], max[1], max[2], max[3]) + val iouThreshold = 0.4f + if (boxIOU(maxRectF, rectF) < iouThreshold) { + pq.add(detection) + } + } + } + return list + } + + // CHANGED: 모델 선택에 따른 전처리 & 후처리 분기 + fun inferenceOD(imageBitmap: Bitmap): ArrayList { + // 전처리 선택 + val floatBuffer = when (odFlavor) { + ODModelFlavor.RTMDET -> bitmapToFloatBufferRtmdet(imageBitmap) // mmdet 기본 전처리 + else -> bitmapToFloatBuffer(imageBitmap) // 기존 YOLOv8 전처리 (/255) + } + + val inputName = session.inputNames.iterator().next() + val shape = longArrayOf( + BATCH_SIZE.toLong(), PIXEL_SIZE.toLong(), + INPUT_SIZE.toLong(), INPUT_SIZE.toLong() + ) + + val inputTensor = OnnxTensor.createTensor(ortEnvironment, floatBuffer, shape) + val ortResult = session.run(Collections.singletonMap(inputName, inputTensor)) + + val result = when (odFlavor) { + ODModelFlavor.RTMDET -> outputsToRTMDetPredictions(ortResult) // NEW + else -> { + // 기존 YOLOv8 경로 그대로 + val outputs = ortResult.get(0).value as Array<*> // [1,84,8400] + outputsToNPMSPredictions(outputs) + } + } + + // 명시적으로 텐서와 결과 메모리 해제 + inputTensor.close() + ortResult.close() + + return result + } + + fun inferencePose(imageBitmap: Bitmap): ArrayList { + val floatBuffer = bitmapToFloatBuffer(imageBitmap) + val inputName = sessionPose.inputNames.iterator().next() + //모델의 요구 입력값 [1 3 640 640] [배치 사이즈, 픽셀(RGB), 너비, 높이], 모델마다 크기는 다를 수 있음. + val shape = longArrayOf( + BATCH_SIZE.toLong(), + PIXEL_SIZE.toLong(), + INPUT_SIZE.toLong(), + INPUT_SIZE.toLong() + ) + val inputTensor = OnnxTensor.createTensor(ortEnvironmentPose, floatBuffer, shape) + val resultTensor = sessionPose.run(Collections.singletonMap(inputName, inputTensor)) + val outputs = resultTensor.get(0).value as Array<*> + val results = outputsToNPMSPredictionsPose(outputs) + + // 명시적으로 텐서와 결과 메모리 해제 + inputTensor.close() + resultTensor.close() + + return results + } + + // 겹치는 비율 (교집합/합집합) + private fun boxIOU(a: RectF, b: RectF): Float { + return boxIntersection(a, b) / boxUnion(a, b) + } + + //교집합 + private fun boxIntersection(a: RectF, b: RectF): Float { + // x1, x2 == 각 rect 객체의 중심 x or y값, w1, w2 == 각 rect 객체의 넓이 or 높이 + val w = overlap( + (a.left + a.right) / 2f, a.right - a.left, + (b.left + b.right) / 2f, b.right - b.left + ) + val h = overlap( + (a.top + a.bottom) / 2f, a.bottom - a.top, + (b.top + b.bottom) / 2f, b.bottom - b.top + ) + + return if (w < 0 || h < 0) 0f else w * h + } + + //합집합 + private fun boxUnion(a: RectF, b: RectF): Float { + val i: Float = boxIntersection(a, b) + + return (a.right - a.left) * (a.bottom - a.top) + (b.right - b.left) * (b.bottom - b.top) - i + } + + //서로 겹치는 부분의 길이 + private fun overlap(x1: Float, w1: Float, x2: Float, w2: Float): Float { + val l1 = x1 - w1 / 2 + val l2 = x2 - w2 / 2 + val r1 = x1 + w1 / 2 + val r2 = x2 + w2 / 2 + + return min(r1, r2) - max(l1, l2) + } +} diff --git a/app/src/main/java/com/a2d2/mobilitygateway/objectDetect/PoseView.kt b/app/src/main/java/com/a2d2/mobilitygateway/objectDetect/PoseView.kt new file mode 100644 index 0000000..992594b --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/objectDetect/PoseView.kt @@ -0,0 +1,209 @@ +package com.a2d2.mobilitygateway.objectDetect + +import android.annotation.SuppressLint +import android.content.Context +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.util.AttributeSet +import android.view.View + +class PoseView(context: Context, attributeSet: AttributeSet) : View(context, attributeSet) { + + private var list: ArrayList? = null + private val pointPaint = Paint().apply { + color = Color.RED + style = Paint.Style.STROKE + strokeWidth = 10f + } + private val linePaint = Paint().apply { + color = Color.YELLOW + style = Paint.Style.FILL + strokeWidth = 5f + } + + fun setList(list: ArrayList) { + this.list = list + } + + @SuppressLint("DrawAllocation") + override fun onDraw(canvas: Canvas) { + //점, 선 그리기 + drawPointsAndLines(canvas) + super.onDraw(canvas) + } + + private fun drawPointsAndLines(canvas: Canvas) { + val scaleX = width / OnnxManager.INPUT_SIZE.toFloat() + val scaleY = scaleX * 9f / 16f + val realY = width * 9f / 16f + val diffY = realY - height + + val kPointsThreshold = 0.35f + list?.forEach { + val points = FloatArray(34) + for ((a, i) in (points.indices step 2).withIndex()) { + if (it[i + 7 + a] > kPointsThreshold) { + points[i] = it[i + 5 + a] * scaleX + points[i + 1] = it[i + 6 + a] * scaleY - (diffY / 2f) + } + } + drawPoint(canvas, points) + drawLines(canvas, points) + } + } + + private fun drawPoint(canvas: Canvas, points: FloatArray) { + for (i in points.indices step 2) { + val xPos = points[i] + val yPos = points[i + 1] + if (xPos > 0 && yPos > 0) { + canvas.drawPoint(xPos, yPos, pointPaint) + } + } + } + + private fun drawLines(canvas: Canvas, points: FloatArray) { + // 점과 점사이에 직선 그리기 + // keypoint 순서 + // 0번 == 코 + // 1번 == 오른쪽 눈 + // 2번 == 왼쪽 눈 + // 3번 == 오른쪽 귀 + // 4번 == 왼쪽 귀 + // 5번 == 오른쪽 어깨 + // 6번 == 왼쪽 어깨 + // 7번 == 오른쪽 팔꿈치 + // 8번 == 왼쪽 팔꿈치 + // 9번 == 오른쪽 손목 + // 10번 == 왼쪽 손목 + // 11번 == 오른쪽 골반 + // 12번 == 왼쪽 골반 + // 13번 == 오른쪽 무릎 + // 14번 == 왼쪽 무릎 + // 15번 == 오른쪽 발 + // 16번 == 왼쪽 발 + + // 코, 오른쪽 눈 연결 + var startX = points[0] + var startY = points[1] + var stopX = points[2] + var stopY = points[3] + drawLine(startX, startY, stopX, stopY, canvas) + // 코, 왼쪽 눈 연결 + startX = points[0] + startY = points[1] + stopX = points[4] + stopY = points[5] + drawLine(startX, startY, stopX, stopY, canvas) + //오른쪽 눈 귀 연결 + startX = points[2] + startY = points[3] + stopX = points[8] + stopY = points[9] + drawLine(startX, startY, stopX, stopY, canvas) + //왼쪽 눈 귀 연결 + startX = points[4] + startY = points[5] + stopX = points[8] + stopY = points[9] + drawLine(startX, startY, stopX, stopY, canvas) + //오른쪽 귀 어깨 연결 + startX = points[6] + startY = points[7] + stopX = points[10] + stopY = points[11] + drawLine(startX, startY, stopX, stopY, canvas) + //왼쪽 귀 어깨 연결 + startX = points[8] + startY = points[9] + stopX = points[12] + stopY = points[13] + drawLine(startX, startY, stopX, stopY, canvas) + //오른쪽 어깨 팔꿈치 연결 + startX = points[10] + startY = points[11] + stopX = points[14] + stopY = points[15] + drawLine(startX, startY, stopX, stopY, canvas) + //왼쪽 어깨 팔꿈치 연결 + startX = points[12] + startY = points[13] + stopX = points[16] + stopY = points[17] + drawLine(startX, startY, stopX, stopY, canvas) + //오른쪽 어깨 골반 연결 + startX = points[10] + startY = points[11] + stopX = points[22] + stopY = points[23] + drawLine(startX, startY, stopX, stopY, canvas) + //왼쪽 어깨 골반 연결 + startX = points[12] + startY = points[13] + stopX = points[24] + stopY = points[25] + drawLine(startX, startY, stopX, stopY, canvas) + //오른쪽 팔꿈치 손목 연결 + startX = points[14] + startY = points[15] + stopX = points[18] + stopY = points[19] + drawLine(startX, startY, stopX, stopY, canvas) + //왼쪽 팔꿈치 손목 연결 + startX = points[16] + startY = points[17] + stopX = points[20] + stopY = points[21] + drawLine(startX, startY, stopX, stopY, canvas) + //오른쪽 골반 무릎 연결 + startX = points[22] + startY = points[23] + stopX = points[26] + stopY = points[27] + drawLine(startX, startY, stopX, stopY, canvas) + //왼쪽 골반 무릎 연결 + startX = points[24] + startY = points[25] + stopX = points[28] + stopY = points[29] + drawLine(startX, startY, stopX, stopY, canvas) + //오른쪽 무릎 발 연결 + startX = points[26] + startY = points[27] + stopX = points[30] + stopY = points[31] + drawLine(startX, startY, stopX, stopY, canvas) + //왼쪽 무릎 발 연결 + startX = points[28] + startY = points[29] + stopX = points[32] + stopY = points[33] + drawLine(startX, startY, stopX, stopY, canvas) + //어깨 좌우 연결 + startX = points[10] + startY = points[11] + stopX = points[12] + stopY = points[13] + drawLine(startX, startY, stopX, stopY, canvas) + //골반 좌우 연결 + startX = points[22] + startY = points[23] + stopX = points[24] + stopY = points[25] + drawLine(startX, startY, stopX, stopY, canvas) + } + + private fun drawLine( + startX: Float, + startY: Float, + stopX: Float, + stopY: Float, + canvas: Canvas + ) { + if (startX > 0 && startY > 0 && stopX > 0 && stopY > 0) { + canvas.drawLine(startX, startY, stopX, stopY, linePaint) + } + } + +} \ No newline at end of file diff --git a/app/src/main/java/com/a2d2/mobilitygateway/objectDetect/RectView.kt b/app/src/main/java/com/a2d2/mobilitygateway/objectDetect/RectView.kt new file mode 100644 index 0000000..1e8ace8 --- /dev/null +++ b/app/src/main/java/com/a2d2/mobilitygateway/objectDetect/RectView.kt @@ -0,0 +1,108 @@ +package com.a2d2.mobilitygateway.objectDetect + +import android.content.Context +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.util.AttributeSet +import android.view.View +import com.a2d2.mobilitygateway.CAMERA_IMAGE_HEIGHT +import com.a2d2.mobilitygateway.CAMERA_IMAGE_WIDTH +import com.a2d2.mobilitygateway.ODModelFlavor +import com.a2d2.mobilitygateway.ODResult +import com.a2d2.mobilitygateway.OD_MODEL +import com.a2d2.mobilitygateway.RTSP_IMAGE_HEIGHT +import com.a2d2.mobilitygateway.RTSP_IMAGE_WIDTH +import com.a2d2.mobilitygateway.use_camera +import kotlin.math.max +import kotlin.math.round + +class RectView(context: Context, attributeSet: AttributeSet) : View(context, attributeSet) { + + private var results: ArrayList? = null + private lateinit var classes: Array + + private val textPaint = Paint().also { + it.textSize = 60f + it.color = Color.WHITE + } + + fun transformRect(results: ArrayList) { + when (OD_MODEL){ + ODModelFlavor.YOLOV8 -> { + // scale 구하기 + val scaleX = width / OnnxManager.INPUT_SIZE.toFloat() + val scaleY = scaleX * 9f / 16f + val realY = width * 9f / 16f + val diffY = realY - height + + results.forEach { + it.rectF.left *= scaleX + it.rectF.right *= scaleX + it.rectF.top = it.rectF.top * scaleY - (diffY / 2f) + it.rectF.bottom = it.rectF.bottom * scaleY - (diffY / 2f) + } + this.results = results + } + ODModelFlavor.RTMDET -> { + results.forEach { + val imgWidth = if(use_camera) CAMERA_IMAGE_WIDTH else RTSP_IMAGE_WIDTH + val imgHeight = if (use_camera) CAMERA_IMAGE_HEIGHT else RTSP_IMAGE_HEIGHT + + val widthRatio : Float = width.toFloat() / OnnxManager.INPUT_SIZE.toFloat() + val heightRatio : Float = height.toFloat() / OnnxManager.INPUT_SIZE.toFloat() + + it.rectF.left *= widthRatio + it.rectF.top *= heightRatio + it.rectF.right *= widthRatio + it.rectF.bottom *= heightRatio + } + this.results = results + } + } + } + + override fun onDraw(canvas: Canvas) { + //그림 그리기 + results?.forEach { +// canvas.drawRect(it.rectF, findPaint(it.classIndex)) + canvas.drawRect(it.rectF.left,it.rectF.top,it.rectF.right,it.rectF.bottom, findPaint(it.classIndex)) + canvas.drawText( + classes[it.classIndex] + ", " + round(it.score * 100) + "%", + it.rectF.left + 10, + it.rectF.top + 60, + textPaint + ) + } + super.onDraw(canvas) + } + + fun setClassLabel(classes: Array) { + this.classes = classes + } + + //paint 지정 + private fun findPaint(classIndex: Int): Paint { + val paint = Paint() + paint.style = Paint.Style.STROKE // 빈 사각형 그림 + paint.strokeWidth = 7.0f // 굵기 7 + paint.strokeCap = Paint.Cap.ROUND // 끝을 뭉특하게 + paint.strokeJoin = Paint.Join.ROUND // 끝 주위도 뭉특하게 + paint.strokeMiter = 100f // 뭉특한 정도는 100도 + + //임의로 지정한 색상 + paint.color = when (classIndex) { + 0, 45, 18, 19, 22, 30, 42, 43, 44, 61, 71, 72 -> Color.WHITE + 11, 12, 13, 14, 25, 37, 38, 79 -> Color.BLUE + 1, 3, 6, 8, 10, 32, 47, 49, 51, 52 -> Color.RED + 23, 46, 48 -> Color.YELLOW + 34, 35, 36, 54, 59, 60, 73, 77, 78 -> Color.GRAY + 24, 26, 27, 28, 62, 64, 65, 66, 67, 68, 69, 74, 75 -> Color.BLACK + 2, 4, 5, 7, 9, 29, 33, 39, 41, 58, 50 -> Color.GREEN + 15, 16, 17, 20, 21, 31, 40, 55, 57, 63 -> Color.DKGRAY + 70, 76 -> Color.LTGRAY + else -> Color.DKGRAY + } + return paint + } +} \ No newline at end of file diff --git a/app/src/main/res/drawable/ic_launcher_background.xml b/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 0000000..07d5da9 --- /dev/null +++ b/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/src/main/res/drawable/ic_launcher_foreground.xml b/app/src/main/res/drawable/ic_launcher_foreground.xml new file mode 100644 index 0000000..2b068d1 --- /dev/null +++ b/app/src/main/res/drawable/ic_launcher_foreground.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/res/layout/activity_engine_view.xml b/app/src/main/res/layout/activity_engine_view.xml new file mode 100644 index 0000000..e6feaa0 --- /dev/null +++ b/app/src/main/res/layout/activity_engine_view.xml @@ -0,0 +1,135 @@ + + + + + + + + + + + + + + + + + + + + + + + + +