Fix hook/deoptimize static methods failed on some Andorid 13 devices

Fix #29

.github/workflows/build.yml

@@ -95,6 +95,7 @@ jobs:
           echo 'android.testoptions.manageddevices.emulator.gpu=swiftshader_indirect' >> gradle.properties
           echo 'android.native.buildOutput=verbose' >> gradle.properties
           echo 'android.sdk.channel=3' >> gradle.properties
+          sudo rm -R /Applications/Xcode*
           ./gradlew :test:allDevicesDebugAndroidTest
           rm -v test/build/outputs/androidTest-results/managedDevice/*/testlog/adb.additional_test_output*
       - name: Upload outputs

lsplant/src/main/jni/art/mirror/class.hpp

@@ -1,9 +1,11 @@
 #pragma once
 
+#include "art/runtime/art_method.hpp"
+#include "art/runtime/handle.hpp"
 #include "common.hpp"
 
 namespace lsplant::art {
-
+class Thread;
 namespace dex {
 class ClassDef {};
 }  // namespace dex
@@ -25,6 +27,75 @@ private:
         return nullptr;
     }
 
+    using BackupMethods = std::list<std::tuple<art::ArtMethod *, void *>>;
+    inline static absl::flat_hash_map<const art::Thread *,
+                                      absl::flat_hash_map<const dex::ClassDef *, BackupMethods>>
+        backup_methods_;
+    inline static std::mutex backup_methods_lock_;
+
+    static void BackupClassMethods(const dex::ClassDef *class_def, art::Thread *self) {
+        std::list<std::tuple<art::ArtMethod *, void *>> out;
+        if (!class_def) return;
+        {
+            std::shared_lock lk(hooked_classes_lock_);
+            if (auto found = hooked_classes_.find(class_def); found != hooked_classes_.end())
+                [[unlikely]] {
+                for (auto method : found->second) {
+                    if (method->IsStatic()) {
+                        LOGV("Backup hooked method %s because of initialization",
+                             method->PrettyMethod(true).data());
+                        out.emplace_back(method, method->GetEntryPoint());
+                    }
+                }
+            }
+        }
+        {
+            std::shared_lock lk(deoptimized_methods_lock_);
+            if (auto found = deoptimized_classes_.find(class_def);
+                found != deoptimized_classes_.end()) [[unlikely]] {
+                for (auto method : found->second) {
+                    if (method->IsStatic()) {
+                        LOGV("Backup deoptimized method %s because of initialization",
+                             method->PrettyMethod(true).data());
+                        out.emplace_back(method, method->GetEntryPoint());
+                    }
+                }
+            }
+        }
+        {
+            std::unique_lock lk(backup_methods_lock_);
+            backup_methods_[self].emplace(class_def, std::move(out));
+        }
+    }
+
+    CREATE_HOOK_STUB_ENTRY(
+        "_ZN3art6mirror5Class9SetStatusENS_6HandleIS1_EENS_11ClassStatusEPNS_6ThreadE", void,
+        SetClassStatus, (Handle<Class> h, uint8_t new_status, Thread *self), {
+            if (new_status == initialized_status) {
+                BackupClassMethods(h->GetClassDef(), self);
+            }
+            return backup(h, new_status, self);
+        });
+
+    CREATE_HOOK_STUB_ENTRY(
+        "_ZN3art6mirror5Class9SetStatusENS_6HandleIS1_EENS1_6StatusEPNS_6ThreadE", void, SetStatus,
+        (Handle<Class> h, int new_status, Thread *self), {
+            if (new_status == static_cast<int>(initialized_status)) {
+                BackupClassMethods(h->GetClassDef(), self);
+            }
+            return backup(h, new_status, self);
+        });
+
+    CREATE_MEM_HOOK_STUB_ENTRY("_ZN3art6mirror5Class9SetStatusENS1_6StatusEPNS_6ThreadE", void,
+                               ClassSetStatus, (Class * thiz, int new_status, Thread *self), {
+                                   if (new_status == static_cast<int>(initialized_status)) {
+                                       BackupClassMethods(thiz->GetClassDef(), self);
+                                   }
+                                   return backup(thiz, new_status, self);
+                               });
+
+    inline static uint8_t initialized_status = 0;
+
 public:
     static bool Init(const HookHandler &handler) {
         if (!RETRIEVE_MEM_FUNC_SYMBOL(GetDescriptor,
@@ -35,6 +106,20 @@ public:
         if (!RETRIEVE_MEM_FUNC_SYMBOL(GetClassDef, "_ZN3art6mirror5Class11GetClassDefEv")) {
             return false;
         }
+
+        if (!HookSyms(handler, SetClassStatus, SetStatus, ClassSetStatus)) {
+            return false;
+        }
+
+        int sdk_int = GetAndroidApiLevel();
+        if (sdk_int >= __ANDROID_API_P__) {
+            initialized_status = 15;
+        } else if (sdk_int == __ANDROID_API_O_MR1__) {
+            initialized_status = 11;
+        } else {
+            initialized_status = 10;
+        }
+
         return true;
     }
 
@@ -54,6 +139,37 @@ public:
         if (GetClassDefSym) return GetClassDef(this);
         return nullptr;
     }
+
+    static auto PopBackup(const dex::ClassDef *class_def, art::Thread *self) {
+        BackupMethods methods;
+        if (!backup_methods_.size()) [[likely]] {
+            return methods;
+        }
+        if (class_def) {
+            std::unique_lock lk(backup_methods_lock_);
+            for (auto it = backup_methods_.begin(); it != backup_methods_.end();) {
+                if (auto found = it->second.find(class_def); found != it->second.end()) {
+                    methods.merge(std::move(found->second));
+                    it->second.erase(found);
+                }
+                if (it->second.empty()) {
+                    backup_methods_.erase(it++);
+                } else {
+                    it++;
+                }
+            }
+        } else if (!class_def && self) {
+            std::unique_lock lk(backup_methods_lock_);
+            if (auto found = backup_methods_.find(self); found != backup_methods_.end()) {
+                for (auto it = found->second.begin(); it != found->second.end();) {
+                    methods.merge(std::move(it->second));
+                    found->second.erase(it++);
+                }
+                backup_methods_.erase(found);
+            }
+        }
+        return methods;
+    }
 };
 
 }  // namespace mirror

lsplant/src/main/jni/art/runtime/class_linker.hpp

@@ -64,41 +64,18 @@ private:
                                (art::ClassLinker * thiz, art::Thread *self, art::ArtMethod *method),
                                { return backup(thiz, self, MayGetBackup(method)); });
 
-    static auto GetBackupMethods(mirror::Class *mirror_class) {
-        std::list<std::tuple<art::ArtMethod *, void *>> out;
-        auto class_def = mirror_class->GetClassDef();
-        if (!class_def) return out;
-        {
-            std::shared_lock lk(hooked_classes_lock_);
-            if (auto found = hooked_classes_.find(class_def); found != hooked_classes_.end())
-                [[unlikely]] {
-                LOGV("Before fixup %s, backup %zu hooked methods' trampoline",
-                     mirror_class->GetDescriptor().c_str(), found->second.size());
-                for (auto method : found->second) {
-                    out.emplace_back(method, method->GetEntryPoint());
-                }
-            }
-        }
-        {
-            std::shared_lock lk(deoptimized_methods_lock_);
-            if (auto found = deoptimized_classes_.find(class_def);
-                found != deoptimized_classes_.end()) [[unlikely]] {
-                LOGV("Before fixup %s, backup %zu deoptimized methods' trampoline",
-                     mirror_class->GetDescriptor().c_str(), found->second.size());
-                for (auto method : found->second) {
-                    out.emplace_back(method, method->GetEntryPoint());
-                }
-            }
-        }
-        return out;
-    }
-
-    static void FixTrampoline(const std::list<std::tuple<art::ArtMethod *, void *>> &methods) {
-        std::shared_lock lk(hooked_methods_lock_);
+    static auto RestoreBackup(const dex::ClassDef *class_def, art::Thread *self) {
+        auto methods = mirror::Class::PopBackup(class_def, self);
         for (const auto &[art_method, old_trampoline] : methods) {
             auto new_trampoline = art_method->GetEntryPoint();
             art_method->SetEntryPoint(old_trampoline);
-            if (IsDeoptimized(art_method)) continue;
+            if (IsDeoptimized(art_method)) {
+                if (new_trampoline != old_trampoline) [[unlikely]] {
+                    LOGV("prevent deoptimized method %s from being overwritten",
+                         art_method->PrettyMethod(true).data());
+                }
+                continue;
+            }
             if (auto backup_method = IsHooked(art_method); backup_method) [[likely]] {
                 if (new_trampoline != old_trampoline) [[unlikely]] {
                     LOGV("propagate entrypoint for %s", backup_method->PrettyMethod(true).data());
@@ -111,26 +88,30 @@ private:
     CREATE_MEM_HOOK_STUB_ENTRY(
         "_ZN3art11ClassLinker22FixupStaticTrampolinesENS_6ObjPtrINS_6mirror5ClassEEE", void,
         FixupStaticTrampolines, (ClassLinker * thiz, ObjPtr<mirror::Class> mirror_class), {
-            auto backup_methods = GetBackupMethods(mirror_class);
             backup(thiz, mirror_class);
-            FixTrampoline(backup_methods);
+            RestoreBackup(mirror_class->GetClassDef(), nullptr);
         });
 
     CREATE_MEM_HOOK_STUB_ENTRY(
         "_ZN3art11ClassLinker22FixupStaticTrampolinesEPNS_6ThreadENS_6ObjPtrINS_6mirror5ClassEEE",
         void, FixupStaticTrampolinesWithThread,
         (ClassLinker * thiz, art::Thread *self, ObjPtr<mirror::Class> mirror_class), {
-            auto backup_methods = GetBackupMethods(mirror_class);
             backup(thiz, self, mirror_class);
-            FixTrampoline(backup_methods);
+            RestoreBackup(mirror_class->GetClassDef(), self);
         });
 
     CREATE_MEM_HOOK_STUB_ENTRY("_ZN3art11ClassLinker22FixupStaticTrampolinesEPNS_6mirror5ClassE",
                                void, FixupStaticTrampolinesRaw,
                                (ClassLinker * thiz, mirror::Class *mirror_class), {
-                                   auto backup_methods = GetBackupMethods(mirror_class);
                                    backup(thiz, mirror_class);
-                                   FixTrampoline(backup_methods);
+                                   RestoreBackup(mirror_class->GetClassDef(), nullptr);
+                               });
+
+    CREATE_MEM_HOOK_STUB_ENTRY(
+        "_ZN3art11ClassLinker26VisiblyInitializedCallback29AdjustThreadVisibilityCounterEPNS_6ThreadEl",
+        void, AdjustThreadVisibilityCounter, (void *thiz, art::Thread *self, ssize_t adjustment), {
+            backup(thiz, self, adjustment);
+            RestoreBackup(nullptr, self);
         });
 
 public:
@@ -147,6 +128,9 @@ public:
             return false;
         }
 
+        // fixup static trampoline may have been inlined
+        HookSyms(handler, AdjustThreadVisibilityCounter);
+
         if (!RETRIEVE_MEM_FUNC_SYMBOL(
                 SetEntryPointsToInterpreter,
                 "_ZNK3art11ClassLinker27SetEntryPointsToInterpreterEPNS_9ArtMethodE"))

lsplant/src/main/jni/art/runtime/handle.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <type_traits>
+
+#include "object_reference.hpp"
+#include "value_object.hpp"
+
+namespace lsplant::art {
+
+namespace mirror {
+class Class;
+};
+
+template <typename T>
+class Handle : public ValueObject {
+public:
+    static_assert(std::is_same_v<T, mirror::Class>, "Expected mirror::Class");
+
+    auto operator->() { return Get(); }
+
+    T *Get() { return down_cast<T *>(reference_->AsMirrorPtr()); }
+
+protected:
+    StackReference<T> *reference_;
+};
+
+}  // namespace lsplant::art

lsplant/src/main/jni/art/runtime/object_reference.hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <cstdint>
+
+namespace lsplant::art {
+template <bool kPoisonReferences, class MirrorType>
+class alignas(4) [[gnu::packed]] ObjectReference {
+    static MirrorType* Decompress(uint32_t ref) {
+        uintptr_t as_bits = kPoisonReferences ? -ref : ref;
+        return reinterpret_cast<MirrorType*>(as_bits);
+    }
+
+    uint32_t reference_;
+
+public:
+    MirrorType* AsMirrorPtr() const { return Decompress(reference_); }
+};
+
+template <class MirrorType>
+class alignas(4) [[gnu::packed]] CompressedReference : public ObjectReference<false, MirrorType> {};
+
+template <class MirrorType>
+class alignas(4) [[gnu::packed]] StackReference : public CompressedReference<MirrorType> {};
+
+template <typename To, typename From>  // use like this: down_cast<T*>(foo);
+inline To down_cast(From* f) {         // so we only accept pointers
+    static_assert(std::is_base_of_v<From, std::remove_pointer_t<To>>,
+                  "down_cast unsafe as To is not a subtype of From");
+
+    return static_cast<To>(f);
+}
+}  // namespace lsplant::art

lsplant/src/main/jni/art/runtime/reflective_handle.hpp

@@ -1,11 +1,13 @@
 #pragma once
 
+#include <type_traits>
+
 #include "reflective_reference.hpp"
+#include "value_object.hpp"
 
 namespace lsplant::art {
 
 class ArtMethod;
-class ValueObject {};
 
 template <typename T>
 class ReflectiveHandle : public ValueObject {

lsplant/src/main/jni/art/runtime/reflective_reference.hpp

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <type_traits>
+
 namespace lsplant::art {
 template <class ReflectiveType>
 class ReflectiveReference {

lsplant/src/main/jni/art/runtime/value_object.hpp

@@ -0,0 +1,5 @@
+#pragma once
+
+namespace lsplant::art {
+class ValueObject {};
+}  // namespace lsplant::art