test new search ssystem

sopel_modules/SpiceBot/Gif.py

@@ -222,6 +222,9 @@ class BotGif():
             url += str(self.valid_api[currentapi]['sfw'])
         # api key
         url += str(self.valid_api[currentapi]['key']) + str(self.valid_api[currentapi]['apikey'])
+        # additional parts
+        if "additional_url" in list(self.valid_api[currentapi].keys()):
+            url += str(self.valid_api[currentapi]['additional_url'])
         return url
 
 

sopel_modules/SpiceBot/Search.py

@@ -9,6 +9,8 @@ from fake_useragent import UserAgent
 from sopel.config.types import StaticSection, ValidatedAttribute
 
 from .Config import config as botconfig
+from .Read import read as botread
+from .Commands import commands as botcommands
 
 # TODO add a cache flush
 
@@ -22,12 +24,27 @@ class Search():
     def __init__(self):
         self.header = {'User-Agent': str(UserAgent().chrome)}
         self.setup_search()
-        self.cache = {
+        valid_search_api_dict = botread.json_to_dict_simple(self.dir_to_scan, "Search API", "SpiceBot_Search")
-                    "info": {},
+
-                    "maps": {},
+        self.cache = {}
-                    "youtube": {},
+
-                    "spiceworks": {},
+        for search_api in list(valid_search_api_dict.keys()):
-                    }
+
+            self.valid_api[search_api] = valid_search_api_dict[search_api]
+            self.valid_api[search_api]["apikey"] = None
+            self.valid_api[search_api]["cache"] = dict()
+
+            apikey = eval("botconfig." + search_api + ".apikey")
+            if apikey:
+                self.valid_api[search_api]["apikey"] = apikey
+
+            self.valid_api[search_api]["comtype"] = "gif_prefix"
+            self.valid_api[search_api]["validcoms"] = [self.valid_api[search_api]["filename"]]
+            botcommands.register(self.valid_api[search_api])
+
+            # check cache
+            if search_api not in list(self.cache.keys()):
+                self.cache[search_api] = dict()
 
     def setup_search(self):
         botconfig.define_section("SpiceBot_Search", SpiceBot_Search_MainSection, validate=False)
@@ -36,9 +53,9 @@ class Search():
 
         # list of defaults
         query_defaults = {
-                        "query": None,
+                        "query": '',
-                        "query_type": "info",
+                        "query_type": "google",
-                        "query_url": None
+                        "nsfw": False,
                         }
 
         # set defaults if they don't exist
@@ -56,28 +73,21 @@ class Search():
         # check cache
         if searchdict["query_type"] not in list(self.cache.keys()):
             self.cache[searchdict["query_type"]] = dict()
-        if searchdict["query_type"] != "custom":
-            if searchdict["query"].lower() in self.cache[searchdict["query_type"]]:
-                return self.cache[searchdict["query_type"]][searchdict["query"]]
 
-        if searchdict["query_type"] == 'maps':
+        if searchdict["query"].lower() in self.cache[searchdict["query_type"]]:
-            returnurl = self.search_maps_google(searchdict)
+            return self.cache[searchdict["query_type"]][searchdict["query"]]
-        elif searchdict["query_type"] == 'youtube':
-            returnurl = self.search_youtube(searchdict)
-        elif searchdict["query_type"] == 'custom':
-            returnurl = self.search_info_custom(searchdict)
-        else:
-            returnurl = self.search_info_google(searchdict)
 
-        if returnurl and searchdict["query_type"] != 'custom':
+        searchdict["searchurl"] = self.search_url_assemble(searchdict)
+        returnurl = self.search_handler(searchdict)
+
+        if returnurl:
             self.cache[searchdict["query_type"]][searchdict["query"].lower()] = str(returnurl)
 
         return returnurl
 
-    def search_info_custom(self, searchdict):
+    def search_handler(self, searchdict):
-        lookfor = searchdict["searchquery"]
         try:
-            var = requests.get(r'' + searchdict["query_url"] + lookfor, headers=self.header)
+            var = requests.get(r'' + searchdict["searchurl"], headers=self.header)
         except Exception as e:
             var = e
             var = None
@@ -85,47 +95,16 @@ class Search():
             return None
         return var.url
 
-    def search_info_google(self, searchdict, retry=False):
+    def search_url_assemble(self, searchdict):
-        lookfor = searchdict["searchquery"]
+        # url base
-        if not botconfig.SpiceBot_Search.search_api or retry:
+        url = str(searchdict)
-            try:
+        # query
-                var = requests.get(r'http://www.google.com/search?q=' + lookfor + '&btnI', headers=self.header)
+        url += str(self.valid_api[searchdict["query_type"]]['query']) + str(searchdict["query"])
-            except Exception as e:
+        # nsfw search? TODO
-                var = e
+        # additional parts
-                var = None
+        if "additional_url" in list(self.valid_api[searchdict["query_type"]].keys()):
-            if not var or not var.url:
+            url += str(self.valid_api[searchdict["query_type"]]['additional_url'])
-                return None
+        return url
-            return var.url
-        else:
-            return self.search_info_google(searchdict, retry=True)
-
-    def search_maps_google(self, searchdict, retry=False):
-        lookfor = searchdict["searchquery"]
-        if not botconfig.SpiceBot_Search.search_api or retry:
-            try:
-                var = requests.get(r'http://www.google.com/maps/place/' + lookfor, headers=self.header)
-            except Exception as e:
-                var = e
-                var = None
-            if not var or not var.url:
-                return None
-            return var.url
-        else:
-            return self.search_maps_google(searchdict, retry=True)
-
-    def search_youtube(self, searchdict, retry=False):
-        lookfor = searchdict["searchquery"]
-        if not botconfig.SpiceBot_Search.search_api or retry:
-            try:
-                var = requests.get(r'https://www.youtube.com/search?q=' + lookfor + '&btnI', headers=self.header)
-            except Exception as e:
-                var = e
-                var = None
-            if not var or not var.url:
-                return None
-            return var.url
-        else:
-            return self.search_youtube(searchdict, retry=True)
 
 
 search = Search()

sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json

@@ -1,8 +0,0 @@
-{
-"?default": {
-              "type": "search",
-              "responses": ["http://community.spiceworks.com/search?query="],
-              "blank_fail": ["https://www.spiceworks.com/"],
-              "search_fail": ["No Spiceworks forum posts found!"],
-              },
-}

sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json

@@ -0,0 +1,4 @@
+{
+"url": "http://www.google.com/maps/place/",
+"query": '',
+}

sopel_modules/SpiceBot_Configs/SpiceBot_Search/google.json

@@ -0,0 +1,5 @@
+{
+"url": "http://www.google.com/search?",
+"query": 'q=',
+"additional_url": "&btnI",
+}

sopel_modules/SpiceBot_Configs/SpiceBot_Search/spiceworks.json

@@ -0,0 +1,4 @@
+{
+"url": "http://community.spiceworks.com/search?",
+"query": 'query=',
+}

sopel_modules/SpiceBot_Configs/SpiceBot_Search/youtube.json

@@ -0,0 +1,5 @@
+{
+"url": "http://www.youtube.com/search?",
+"query": 'q=',
+"additional_url": "&btnI",
+}

sopel_modules/SpiceBot_Core_Commands/AI.py

@@ -176,7 +176,7 @@ def bot_command_rule_ai(bot, trigger):
             if searchterm:
                 if trigger_args[0].lower() == "where":
                     searchdict = {
-                                    "type": "maps",
+                                    "type": "gmaps",
                                     "query": searchterm,
                                     }
                     searchreturn = SpiceBot.search.search(searchdict)

sopel_modules/SpiceBot_Core_Commands/DictComs.py

@@ -298,12 +298,11 @@ def bot_dictcom_search(bot, trigger, botcom):
         botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = botcom.dict["dict"][botcom.dict["responsekey"]]["blank_fail"]
         return bot_dictcom_reply_shared(bot, trigger, botcom)
 
-    searchurl = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0]
+    query_type = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0]
     searchterm = botcom.dict["completestring"]
 
     searchdict = {
-                    "query_type": "custom",
+                    "query_type": query_type,
-                    "query_url": searchurl,
                     "query": searchterm,
                     }
 

sopel_modules/SpiceBot_Core_Tools/Search.py

@@ -11,7 +11,7 @@ import sopel_modules.SpiceBot as SpiceBot
 
 
 @SpiceBot.prerun('module')
-@sopel.module.commands('search', 'find', 'google', 'lookup')
+@sopel.module.commands('search', 'find', 'googled', 'lookup')
 def search_main(bot, trigger, botcom):
 
     searchterm = spicemanip(botcom.dict['args'], 0) or None
@@ -40,7 +40,7 @@ def search_where(bot, trigger, botcom):
         return
     searchdict = {
                     "query": searchterm,
-                    "query_type": "maps",
+                    "query_type": "gmaps",
                     }
 
     searchreturn = SpiceBot.search.search(searchdict)
@@ -70,7 +70,7 @@ def search_what(bot, trigger, botcom):
 
 
 @SpiceBot.prerun('module')
-@sopel.module.commands("youtube", "video")
+@sopel.module.commands("youtubes", "video")
 def search_video(bot, trigger, botcom):
 
     searchterm = spicemanip(botcom.dict['args'], 0) or None