fix: linter issues

src/benchmark/vidvec/video_vec_es_indexer.py

@@ -72,7 +72,7 @@ def store_video_vector():
         embedding = vid_vec_rep_resnet.run(video)
         doc = generate_document(file_name, embedding)
         # TODO: save doc to storage so we don't have to recompute embeddings again
-        result = es.store(media_type, doc)
+        es.store(media_type, doc)
         count += 1
         print("Indexed file:", count)
         # print("result:", result)

src/core/config.py

@@ -7,13 +7,11 @@
 @dennyabrain used dacite as a quick way to convert nested dicts to python dataclasess. While I am happy with its performance so far, I am happy to consider a different solution while also accounting for data validation, which is not done as of now.
 """
 import logging
-
-log = logging.getLogger(__name__)
-
 from typing import List, Optional
 import yaml
 from dataclasses import dataclass
 from dacite import from_dict
+log = logging.getLogger(__name__)
 
 
 @dataclass

src/core/feluda.py

@@ -1,9 +1,7 @@
 import logging
-
-log = logging.getLogger(__name__)
-
 from core import config
 from enum import Enum
+log = logging.getLogger(__name__)
 
 
 class ComponentType(Enum):

src/core/models/media.py

@@ -17,13 +17,13 @@ class MediaType(Enum):
 
     @classmethod
     def make(media_type):
-        if media_type is "text":
+        if media_type == "text":
             return MediaType.TEXT
-        elif media_type is "image":
+        elif media_type == "image":
             return MediaType.IMAGE
-        elif media_type is "video":
+        elif media_type == "video":
             return MediaType.VIDEO
-        elif media_type is "audio":
+        elif media_type == "audio":
             return MediaType.AUDIO
         else:
             return MediaType.UNSUPPORTED

src/core/models/media_factory.py

@@ -69,7 +69,7 @@ def make_from_url(video_url):
             wget.download(video_url, out=fname)
             print("video downloaded")
         except Exception as e:
-            log.exception("Error downloading video")
+            log.exception("Error downloading video:", e)
             raise Exception("Error Downloading Video")
         return {"path": fname}
 
@@ -92,7 +92,7 @@ def make_from_url(audio_url):
             wget.download(audio_url, out=audio_file)
             print("audio downloaded")
         except Exception as e:
-            log.exception("Error downloading audio")
+            log.exception("Error downloading audio:", e)
             raise Exception("Error Downloading audio")
         return {"path": audio_file}
 

src/core/operators/__init__.py

@@ -1,8 +1,7 @@
 import logging
-
-log = logging.getLogger(__name__)
 import importlib
 from core.config import OperatorConfig
+log = logging.getLogger(__name__)
 
 PACKAGE = "core.operators"
 

src/core/operators/audio_cnn_model/config.py

@@ -1,5 +1,5 @@
 import os
-import numpy as np
+#import numpy as np
 import csv
 from pathlib import Path
 

src/core/operators/audio_cnn_model/inference.py

@@ -1,13 +1,13 @@
 import os
-import numpy as np
-import argparse
-import librosa
-import matplotlib.pyplot as plt
+# import numpy as np
+# import argparse
+# import librosa
+# import matplotlib.pyplot as plt
 import torch
 from pathlib import Path
 
 from .pytorch_utils import move_data_to_device
-from .models import Cnn14, Cnn14_DecisionLevelMax
+from .models import Cnn14 #, Cnn14_DecisionLevelMax
 from .config import labels, classes_num
 
 

src/core/operators/audio_cnn_model/models.py

@@ -1,18 +1,18 @@
 """This models.py contains selected models from: 
 https://github.com/qiuqiangkong/audioset_tagging_cnn/blob/master/pytorch/models.py
 """
-import os
-import sys
-import math
-import time
-import numpy as np
-import matplotlib.pyplot as plt
+# import os
+# import sys
+# import math
+# import time
+# import numpy as np
+# import matplotlib.pyplot as plt
 
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
-import torch.utils.checkpoint as cp
-from torch.nn.parameter import Parameter
+# import torch.utils.checkpoint as cp
+# from torch.nn.parameter import Parameter
 
 from torchlibrosa.stft import Spectrogram, LogmelFilterBank
 from torchlibrosa.augmentation import SpecAugmentation

src/core/operators/audio_cnn_model/pytorch_utils.py

@@ -3,7 +3,7 @@
 """
 import torch
 import torch.nn as nn
-import torch.nn.functional as F
+# import torch.nn.functional as F
 
 
 def move_data_to_device(x, device):

src/core/operators/audio_vec_embedding.py

@@ -7,7 +7,7 @@ def initialize(param):
     import librosa
     # from panns_inference import AudioTagging
     from core.operators.audio_cnn_model.inference import AudioTagging
-    import os
+    # import os
 
     # load the default model into cpu.
     model = AudioTagging(checkpoint_path=None, device='cpu')

src/core/operators/detect_text_in_image.py

@@ -43,7 +43,8 @@ def initialize(param):
     global vision, json, os, boto3, MessageToJson, GOOGLE_API_KEY
 
     from google.cloud import vision
-    import json, os
+    import json
+    import os
     import boto3
     from google.protobuf.json_format import MessageToJson
 

src/core/operators/test_detect_objects.py

@@ -1,5 +1,5 @@
 import unittest
-from unittest.mock import patch
+# from unittest.mock import patch
 import detect_objects
 
 class Test(unittest.TestCase):

src/core/operators/test_image_vec_rep_resnet.py

@@ -1,5 +1,5 @@
 import unittest
-from unittest.case import skip
+# from unittest.case import skip
 from core.operators import image_vec_rep_resnet
 from core.models.media_factory import ImageFactory
 

src/core/operators/test_md5_hash.py

@@ -1,5 +1,5 @@
 import unittest
-from unittest.case import skip
+# from unittest.case import skip
 from core.operators import md5_hash
 from core.models.media_factory import VideoFactory
 

src/core/operators/vid_vec_rep_resnet.py

@@ -159,7 +159,7 @@ def get_video_attributes(self, v):
 
         def extract_frames(self, v):
             # print("extracting frames")
-            images = []
+            # images = []
             for i in range(self.n_frames):
                 success, image = v.read()
                 if image is None:

src/core/queue/__init__.py

@@ -1,9 +1,9 @@
 import logging
-
-log = logging.getLogger(__name__)
 from . import rabbit_mq
 from core.config import QueueConfig
-from os import environ
+# from os import environ
+
+log = logging.getLogger(__name__)
 
 queues = {"rabbitmq": rabbit_mq.RabbitMQ}
 

src/core/queue/rabbit_mq.py

@@ -1,11 +1,11 @@
 import logging
 from core.config import QueueConfig
-
-log = logging.getLogger(__name__)
 import pika
 import json
 from os import environ
 
+log = logging.getLogger(__name__)
+
 
 class RabbitMQ:
     def __init__(self, param: QueueConfig):

src/core/server.py

@@ -1,11 +1,11 @@
 import logging
 
 from core.config import ServerConfig
-
-log = logging.getLogger(__name__)
 from flask import Flask
 from flask_cors import CORS
 
+log = logging.getLogger(__name__)
+
 
 class Server:
     def __init__(self, param: ServerConfig) -> None:
@@ -28,7 +28,7 @@ def enable_endpoints(self):
                     endpoint, name, methods = route
                     self.app.add_url_rule(endpoint, name, handler, methods=methods)
             except Exception as e:
-                log.exception("Could not add Route")
+                log.exception("Could not add Route:", e)
 
     def start(self):
         @self.app.route("/")

src/core/store/es_vec.py

@@ -1,8 +1,6 @@
 import logging
 from core.models.media import MediaType
 from core.config import StoreConfig
-
-log = logging.getLogger(__name__)
 from elasticsearch import Elasticsearch, helpers as eshelpers
 from .es_vec_mappings import mappings
 from .es_vec_adapter import es_to_sanitized
@@ -11,6 +9,8 @@
 import os
 import json
 
+log = logging.getLogger(__name__)
+
 class ES:
     def __init__(self, config: StoreConfig):
         # self.es_host = config.parameters.host_name

src/endpoint/health.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+# from dataclasses import dataclass
 from flask import request
 import json
 

src/endpoint/index/__init__.py

@@ -1,5 +1,5 @@
 # from dataclasses import dataclass
-from . import endpoint, handler, model
+# from . import endpoint, handler, model
 
 
 # @dataclass

src/endpoint/index/handler.py

@@ -1,14 +1,14 @@
-from pydoc import plain
-from typing import Callable
-from core.models.media import MediaMode, MediaType
+# from pydoc import plain
+# from typing import Callable
+from core.models.media import MediaType # MediaMode,
 from core.feluda import Feluda
-from core.store.es_vec_adapter import text_rep_to_es_doc
+# from core.store.es_vec_adapter import text_rep_to_es_doc
 from .model import ConfigMode, Post
 from flask import request
-import json
+# import json
 from typing import Callable
 import logging
-import inspect
+# import inspect
 from datetime import datetime
 
 log = logging.getLogger(__name__)
@@ -98,8 +98,8 @@ def index(self, generateRepresentation: Callable):
             else:
                 return "Unexpected Index Mode", 400
 
-        except:
-            log.exception("Unable to handle Index Request")
+        except Exception as e:
+            log.exception("Unable to handle Index Request:", e)
             return "Unable to handle Index Request", 400
 
     def make_handler(self):

src/endpoint/index/test.py

@@ -19,23 +19,31 @@ class IndexResult(Enum):
     FAILURE = 2
 
 
+class IndexHandler:
+    @staticmethod
+    def make(media_type: MediaType, mode: MediaMode):
+        if media_type is MediaType.TEXT:
+            # TODO: below line commented out due to error
+            # return IndexTextHandler.make(mode)
+            return
+        # elif media_type is MediaType.TEXT:
+        #     # TODO: below line commented out due to error
+        #     # return IndexTextHandler.make(mode)
+        #     return
+        # elif media_type is MediaType.TEXT:
+        #     # TODO: below line commented out due to error
+        #     # return IndexTextHandler.make(mode)
+        #     return
+        else:
+            return AttributeError
+
+
 payload = {"post": {}, "metadata": {}, "config": {}}
 post = Post.fromRequestPayload(payload["post"]["media_type"], payload)
 
 # mode can be 'file' or 'url'
 index_handler = IndexHandler.make(media_type=post.type, mode=MediaMode.URL)
 result = index_handler(post)
-add_to_report_queue(result)
+# TODO: below line commented out due to error
+# add_to_report_queue(result)
 
-
-class IndexHandler:
-    @staticmethod
-    def make(media_type: MediaType, mode: MediaMode):
-        if media_type is MediaType.TEXT:
-            return IndexTextHandler.make(mode)
-        elif media_type is MediaType.TEXT:
-            return IndexTextHandler.make(mode)
-        elif media_type is MediaType.TEXT:
-            return IndexTextHandler.make(mode)
-        else:
-            return AttributeError

src/endpoint/index/test_index.py

@@ -1,5 +1,5 @@
 import unittest
-from unittest.case import skip
+# from unittest.case import skip
 import requests
 import json
 

src/endpoint/search.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+# from dataclasses import dataclass
 from core.models.media import MediaType
 from core.models.media_factory import media_factory
 from flask import request
@@ -63,8 +63,8 @@ def handle_search(self):
                     return {"message": "Unsupported Query Type"}
             else:
                 return "Unable to handle Index Request", 400
-        except:
-            log.exception("Unable to handle Index Request")
+        except Exception as e:
+            log.exception("Unable to handle Index Request:", e)
             return "Unable to handle Index Request", 400
 
     def make_handlers(self):

src/indexer.py

@@ -3,7 +3,7 @@
 from core.feluda import ComponentType, Feluda
 from core.logger import Logger
 import json
-import os
+# import os
 
 log = Logger(__name__)
 
@@ -31,7 +31,7 @@ def worker(ch, method, properties, body):
             representation = generateRepresentation(post, operators)
             document = generateDocument(post, representation)
             # print("-----> 2", document)
-            save_result = feluda.store.store(post.type, document)
+            feluda.store.store(post.type, document)
             # return save_result
             report = make_report(data, "indexed")
 
@@ -59,4 +59,4 @@ def worker(ch, method, properties, body):
     feluda.start_component(ComponentType.QUEUE)
     feluda.queue.listen("tattle-search-index-queue", indexer(feluda))
 except Exception as e:
-    log.exception("Error Initializing Indexer")
+    log.exception("Error Initializing Indexer:", e)