edit : hugging face login 로컬에 로그인되어있으면 무시

custom_apps/FEATURE_VECTOR_SIMILARITY_FAISS/fem_openaiclipvit.py

@@ -31,6 +31,8 @@ import os, sys
 
 from transformers import CLIPProcessor, CLIPModel
 from huggingface_hub import login as huggingface_login
+from huggingface_hub import whoami, logout
+from huggingface_hub.utils import LocalTokenNotFoundError
 
 """
 Package: custom
@@ -104,7 +106,18 @@ class FEOpenAIClipViT(FEM.FeatureExtractionModel):
 
         # huggingface token
         if self.huggingface_token:
-            huggingface_login(fem_arguments.token)
+            """
+            토큰이 있다면 로그인이 되어있는지 확인
+            안되어있다면 로그인 시도
+            """
+            try:
+                user_info = whoami()
+            except Exception as LocalTokenNotFoundError:
+                huggingface_login(fem_arguments.token)
+                
+            except Exception as e:
+                log.error(f'Huggingface login error: {e}')
+                raise e
 
         # model path check
         if not os.path.exists(fem_arguments.trained_model):

make_vector_files.py

@@ -1,10 +1,18 @@
-# from custom_apps.FEATURE_VECTOR_SIMILARITY_FAISS.faiss_functions import get_clip_info
 import os
+import logging
+import matplotlib.pyplot as plt
+
 from custom_apps.FEATURE_VECTOR_SIMILARITY_FAISS.faiss_similarity_search import VectorSimilarity
 from custom_apps.FEATURE_VECTOR_SIMILARITY_FAISS.const import *
 from custom_apps.FEATURE_VECTOR_SIMILARITY_FAISS.faiss_functions import get_models, find_glass_folder_images, find_parts_folder_images
 from vector_rest.app import models as VM
 
+
+#log level
+os.environ["HF_HUB_VERBOSITY"] = "info"
+matplotlib_logger = logging.getLogger("matplotlib")
+matplotlib_logger.setLevel(logging.INFO)
+
 def make_image_files(item_info, index_type:VM.VitIndexType, model_type:VM.VitModelType):
     model = get_models(index_type=index_type, model_type=model_type)
     
@@ -53,7 +61,7 @@ def make_vector_files(item_info, index_type:VM.VitIndexType, model_type:VM.VitMo
     
     txt_file_path = make_image_files(item_info=item_info, index_type=index_type, model_type=model_type)
     
-    index_file_path = os.path.join(IMG_LIST_PATH,f'{model.name}_{os.path.basename(model.value[1].trained_model)}_{model.value[1].index_type}_{item_info}_{DEFAULT_INDEX_NAME_SUFFIX}')
+    index_file_path = os.path.join(FAISS_VECTOR_PATH,f'{model.name}_{os.path.basename(model.value[1].trained_model)}_{model.value[1].index_type}_{item_info}_{DEFAULT_INDEX_NAME_SUFFIX}')
     
     if not os.path.exists(index_file_path):
         
@@ -66,12 +74,11 @@ def make_vector_files(item_info, index_type:VM.VitIndexType, model_type:VM.VitMo
             
         vector_model.save_index_from_files(images_path_lists=image_lists,
                                 save_index_dir=FAISS_VECTOR_PATH,
-                                save_txt_dir=IMG_LIST_PATH,
                                 item_info=item_info,
                                 index_type=model.value[1].index_type)
         
     else:
-        temp_path = os.path.join(IMG_LIST_PATH,f'{model.name}_{os.path.basename(model.value[1].trained_model)}_{model.value[1].index_type}_{item_info}_{DEFAULT_INDEX_NAME_SUFFIX}.bak')
+        temp_path = os.path.join(FAISS_VECTOR_PATH,f'{model.name}_{os.path.basename(model.value[1].trained_model)}_{model.value[1].index_type}_{item_info}_{DEFAULT_INDEX_NAME_SUFFIX}.bak')
         
         try:
             os.rename(index_file_path, temp_path)
@@ -85,7 +92,6 @@ def make_vector_files(item_info, index_type:VM.VitIndexType, model_type:VM.VitMo
                 
             vector_model.save_index_from_files(images_path_lists=image_lists,
                                     save_index_dir=FAISS_VECTOR_PATH,
-                                    save_txt_dir=IMG_LIST_PATH,
                                     item_info=item_info,
                                     index_type=model.value[1].index_type)
         except Exception as e:
@@ -102,7 +108,6 @@ def make_vector_files(item_info, index_type:VM.VitIndexType, model_type:VM.VitMo
 
 if __name__ == '__main__':
     
-    import time
     class_attributes = dict(VectorSearchItem.__dict__)
 
     pure_data_dict = {
@@ -114,4 +119,4 @@ if __name__ == '__main__':
     for item_key, item_value in pure_data_dict.items():
         make_vector_files(item_info=item_value, index_type=VM.VitIndexType.l2, model_type=VM.VitModelType.b32)
         
-        time.sleep(5) # huggingface api 요청 제한 회피 위해 대기 TODO(jwkim) huggingface 로그인은 한번만 진행하게 변경
+        # time.sleep(5) # huggingface api 요청 제한 회피 위해 대기 TODO(jwkim) huggingface 로그인은 한번만 진행하게 변경