From 2647b2a52228c7605bc5dbf4a356e92d0f3cc268 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 14:32:43 -0400 Subject: [PATCH] test new search ssystem --- sopel_modules/SpiceBot/Gif.py | 3 + sopel_modules/SpiceBot/Search.py | 109 +++++++----------- .../SpiceBot_DictComs/search/spiceworks.json | 8 -- .../SpiceBot_Search/gmaps.json | 4 + .../SpiceBot_Search/google.json | 5 + .../SpiceBot_Search/spiceworks.json | 4 + .../SpiceBot_Search/youtube.json | 5 + sopel_modules/SpiceBot_Core_Commands/AI.py | 2 +- .../SpiceBot_Core_Commands/DictComs.py | 5 +- sopel_modules/SpiceBot_Core_Tools/Search.py | 6 +- 10 files changed, 71 insertions(+), 80 deletions(-) delete mode 100644 sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json create mode 100644 sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json create mode 100644 sopel_modules/SpiceBot_Configs/SpiceBot_Search/google.json create mode 100644 sopel_modules/SpiceBot_Configs/SpiceBot_Search/spiceworks.json create mode 100644 sopel_modules/SpiceBot_Configs/SpiceBot_Search/youtube.json diff --git a/sopel_modules/SpiceBot/Gif.py b/sopel_modules/SpiceBot/Gif.py index 2a725b9..ea9ff51 100644 --- a/sopel_modules/SpiceBot/Gif.py +++ b/sopel_modules/SpiceBot/Gif.py @@ -222,6 +222,9 @@ class BotGif(): url += str(self.valid_api[currentapi]['sfw']) # api key url += str(self.valid_api[currentapi]['key']) + str(self.valid_api[currentapi]['apikey']) + # additional parts + if "additional_url" in list(self.valid_api[currentapi].keys()): + url += str(self.valid_api[currentapi]['additional_url']) return url diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index 7e51827..81d0634 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -9,6 +9,8 @@ from fake_useragent import UserAgent from sopel.config.types import StaticSection, ValidatedAttribute from .Config import config as botconfig +from .Read import read as botread +from .Commands import commands as botcommands # TODO add a cache flush @@ -22,12 +24,27 @@ class Search(): def __init__(self): self.header = {'User-Agent': str(UserAgent().chrome)} self.setup_search() - self.cache = { - "info": {}, - "maps": {}, - "youtube": {}, - "spiceworks": {}, - } + valid_search_api_dict = botread.json_to_dict_simple(self.dir_to_scan, "Search API", "SpiceBot_Search") + + self.cache = {} + + for search_api in list(valid_search_api_dict.keys()): + + self.valid_api[search_api] = valid_search_api_dict[search_api] + self.valid_api[search_api]["apikey"] = None + self.valid_api[search_api]["cache"] = dict() + + apikey = eval("botconfig." + search_api + ".apikey") + if apikey: + self.valid_api[search_api]["apikey"] = apikey + + self.valid_api[search_api]["comtype"] = "gif_prefix" + self.valid_api[search_api]["validcoms"] = [self.valid_api[search_api]["filename"]] + botcommands.register(self.valid_api[search_api]) + + # check cache + if search_api not in list(self.cache.keys()): + self.cache[search_api] = dict() def setup_search(self): botconfig.define_section("SpiceBot_Search", SpiceBot_Search_MainSection, validate=False) @@ -36,9 +53,9 @@ class Search(): # list of defaults query_defaults = { - "query": None, - "query_type": "info", - "query_url": None + "query": '', + "query_type": "google", + "nsfw": False, } # set defaults if they don't exist @@ -56,28 +73,21 @@ class Search(): # check cache if searchdict["query_type"] not in list(self.cache.keys()): self.cache[searchdict["query_type"]] = dict() - if searchdict["query_type"] != "custom": - if searchdict["query"].lower() in self.cache[searchdict["query_type"]]: - return self.cache[searchdict["query_type"]][searchdict["query"]] - if searchdict["query_type"] == 'maps': - returnurl = self.search_maps_google(searchdict) - elif searchdict["query_type"] == 'youtube': - returnurl = self.search_youtube(searchdict) - elif searchdict["query_type"] == 'custom': - returnurl = self.search_info_custom(searchdict) - else: - returnurl = self.search_info_google(searchdict) + if searchdict["query"].lower() in self.cache[searchdict["query_type"]]: + return self.cache[searchdict["query_type"]][searchdict["query"]] - if returnurl and searchdict["query_type"] != 'custom': + searchdict["searchurl"] = self.search_url_assemble(searchdict) + returnurl = self.search_handler(searchdict) + + if returnurl: self.cache[searchdict["query_type"]][searchdict["query"].lower()] = str(returnurl) return returnurl - def search_info_custom(self, searchdict): - lookfor = searchdict["searchquery"] + def search_handler(self, searchdict): try: - var = requests.get(r'' + searchdict["query_url"] + lookfor, headers=self.header) + var = requests.get(r'' + searchdict["searchurl"], headers=self.header) except Exception as e: var = e var = None @@ -85,47 +95,16 @@ class Search(): return None return var.url - def search_info_google(self, searchdict, retry=False): - lookfor = searchdict["searchquery"] - if not botconfig.SpiceBot_Search.search_api or retry: - try: - var = requests.get(r'http://www.google.com/search?q=' + lookfor + '&btnI', headers=self.header) - except Exception as e: - var = e - var = None - if not var or not var.url: - return None - return var.url - else: - return self.search_info_google(searchdict, retry=True) - - def search_maps_google(self, searchdict, retry=False): - lookfor = searchdict["searchquery"] - if not botconfig.SpiceBot_Search.search_api or retry: - try: - var = requests.get(r'http://www.google.com/maps/place/' + lookfor, headers=self.header) - except Exception as e: - var = e - var = None - if not var or not var.url: - return None - return var.url - else: - return self.search_maps_google(searchdict, retry=True) - - def search_youtube(self, searchdict, retry=False): - lookfor = searchdict["searchquery"] - if not botconfig.SpiceBot_Search.search_api or retry: - try: - var = requests.get(r'https://www.youtube.com/search?q=' + lookfor + '&btnI', headers=self.header) - except Exception as e: - var = e - var = None - if not var or not var.url: - return None - return var.url - else: - return self.search_youtube(searchdict, retry=True) + def search_url_assemble(self, searchdict): + # url base + url = str(searchdict) + # query + url += str(self.valid_api[searchdict["query_type"]]['query']) + str(searchdict["query"]) + # nsfw search? TODO + # additional parts + if "additional_url" in list(self.valid_api[searchdict["query_type"]].keys()): + url += str(self.valid_api[searchdict["query_type"]]['additional_url']) + return url search = Search() diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json b/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json deleted file mode 100644 index 1fc801a..0000000 --- a/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json +++ /dev/null @@ -1,8 +0,0 @@ -{ -"?default": { - "type": "search", - "responses": ["http://community.spiceworks.com/search?query="], - "blank_fail": ["https://www.spiceworks.com/"], - "search_fail": ["No Spiceworks forum posts found!"], - }, -} diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json new file mode 100644 index 0000000..34b84c3 --- /dev/null +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json @@ -0,0 +1,4 @@ +{ +"url": "http://www.google.com/maps/place/", +"query": '', +} diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/google.json b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/google.json new file mode 100644 index 0000000..f9f3bf5 --- /dev/null +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/google.json @@ -0,0 +1,5 @@ +{ +"url": "http://www.google.com/search?", +"query": 'q=', +"additional_url": "&btnI", +} diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/spiceworks.json b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/spiceworks.json new file mode 100644 index 0000000..31f9c1d --- /dev/null +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/spiceworks.json @@ -0,0 +1,4 @@ +{ +"url": "http://community.spiceworks.com/search?", +"query": 'query=', +} diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/youtube.json b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/youtube.json new file mode 100644 index 0000000..220eb26 --- /dev/null +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/youtube.json @@ -0,0 +1,5 @@ +{ +"url": "http://www.youtube.com/search?", +"query": 'q=', +"additional_url": "&btnI", +} diff --git a/sopel_modules/SpiceBot_Core_Commands/AI.py b/sopel_modules/SpiceBot_Core_Commands/AI.py index 3dbf8c8..1e7b928 100644 --- a/sopel_modules/SpiceBot_Core_Commands/AI.py +++ b/sopel_modules/SpiceBot_Core_Commands/AI.py @@ -176,7 +176,7 @@ def bot_command_rule_ai(bot, trigger): if searchterm: if trigger_args[0].lower() == "where": searchdict = { - "type": "maps", + "type": "gmaps", "query": searchterm, } searchreturn = SpiceBot.search.search(searchdict) diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index dbad838..9a5e9e3 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -298,12 +298,11 @@ def bot_dictcom_search(bot, trigger, botcom): botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = botcom.dict["dict"][botcom.dict["responsekey"]]["blank_fail"] return bot_dictcom_reply_shared(bot, trigger, botcom) - searchurl = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0] + query_type = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0] searchterm = botcom.dict["completestring"] searchdict = { - "query_type": "custom", - "query_url": searchurl, + "query_type": query_type, "query": searchterm, } diff --git a/sopel_modules/SpiceBot_Core_Tools/Search.py b/sopel_modules/SpiceBot_Core_Tools/Search.py index 42419ad..24f96e6 100644 --- a/sopel_modules/SpiceBot_Core_Tools/Search.py +++ b/sopel_modules/SpiceBot_Core_Tools/Search.py @@ -11,7 +11,7 @@ import sopel_modules.SpiceBot as SpiceBot @SpiceBot.prerun('module') -@sopel.module.commands('search', 'find', 'google', 'lookup') +@sopel.module.commands('search', 'find', 'googled', 'lookup') def search_main(bot, trigger, botcom): searchterm = spicemanip(botcom.dict['args'], 0) or None @@ -40,7 +40,7 @@ def search_where(bot, trigger, botcom): return searchdict = { "query": searchterm, - "query_type": "maps", + "query_type": "gmaps", } searchreturn = SpiceBot.search.search(searchdict) @@ -70,7 +70,7 @@ def search_what(bot, trigger, botcom): @SpiceBot.prerun('module') -@sopel.module.commands("youtube", "video") +@sopel.module.commands("youtubes", "video") def search_video(bot, trigger, botcom): searchterm = spicemanip(botcom.dict['args'], 0) or None