From 2647b2a52228c7605bc5dbf4a356e92d0f3cc268 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 14:32:43 -0400 Subject: [PATCH 01/36] test new search ssystem --- sopel_modules/SpiceBot/Gif.py | 3 + sopel_modules/SpiceBot/Search.py | 109 +++++++----------- .../SpiceBot_DictComs/search/spiceworks.json | 8 -- .../SpiceBot_Search/gmaps.json | 4 + .../SpiceBot_Search/google.json | 5 + .../SpiceBot_Search/spiceworks.json | 4 + .../SpiceBot_Search/youtube.json | 5 + sopel_modules/SpiceBot_Core_Commands/AI.py | 2 +- .../SpiceBot_Core_Commands/DictComs.py | 5 +- sopel_modules/SpiceBot_Core_Tools/Search.py | 6 +- 10 files changed, 71 insertions(+), 80 deletions(-) delete mode 100644 sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json create mode 100644 sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json create mode 100644 sopel_modules/SpiceBot_Configs/SpiceBot_Search/google.json create mode 100644 sopel_modules/SpiceBot_Configs/SpiceBot_Search/spiceworks.json create mode 100644 sopel_modules/SpiceBot_Configs/SpiceBot_Search/youtube.json diff --git a/sopel_modules/SpiceBot/Gif.py b/sopel_modules/SpiceBot/Gif.py index 2a725b9..ea9ff51 100644 --- a/sopel_modules/SpiceBot/Gif.py +++ b/sopel_modules/SpiceBot/Gif.py @@ -222,6 +222,9 @@ class BotGif(): url += str(self.valid_api[currentapi]['sfw']) # api key url += str(self.valid_api[currentapi]['key']) + str(self.valid_api[currentapi]['apikey']) + # additional parts + if "additional_url" in list(self.valid_api[currentapi].keys()): + url += str(self.valid_api[currentapi]['additional_url']) return url diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index 7e51827..81d0634 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -9,6 +9,8 @@ from fake_useragent import UserAgent from sopel.config.types import StaticSection, ValidatedAttribute from .Config import config as botconfig +from .Read import read as botread +from .Commands import commands as botcommands # TODO add a cache flush @@ -22,12 +24,27 @@ class Search(): def __init__(self): self.header = {'User-Agent': str(UserAgent().chrome)} self.setup_search() - self.cache = { - "info": {}, - "maps": {}, - "youtube": {}, - "spiceworks": {}, - } + valid_search_api_dict = botread.json_to_dict_simple(self.dir_to_scan, "Search API", "SpiceBot_Search") + + self.cache = {} + + for search_api in list(valid_search_api_dict.keys()): + + self.valid_api[search_api] = valid_search_api_dict[search_api] + self.valid_api[search_api]["apikey"] = None + self.valid_api[search_api]["cache"] = dict() + + apikey = eval("botconfig." + search_api + ".apikey") + if apikey: + self.valid_api[search_api]["apikey"] = apikey + + self.valid_api[search_api]["comtype"] = "gif_prefix" + self.valid_api[search_api]["validcoms"] = [self.valid_api[search_api]["filename"]] + botcommands.register(self.valid_api[search_api]) + + # check cache + if search_api not in list(self.cache.keys()): + self.cache[search_api] = dict() def setup_search(self): botconfig.define_section("SpiceBot_Search", SpiceBot_Search_MainSection, validate=False) @@ -36,9 +53,9 @@ class Search(): # list of defaults query_defaults = { - "query": None, - "query_type": "info", - "query_url": None + "query": '', + "query_type": "google", + "nsfw": False, } # set defaults if they don't exist @@ -56,28 +73,21 @@ class Search(): # check cache if searchdict["query_type"] not in list(self.cache.keys()): self.cache[searchdict["query_type"]] = dict() - if searchdict["query_type"] != "custom": - if searchdict["query"].lower() in self.cache[searchdict["query_type"]]: - return self.cache[searchdict["query_type"]][searchdict["query"]] - if searchdict["query_type"] == 'maps': - returnurl = self.search_maps_google(searchdict) - elif searchdict["query_type"] == 'youtube': - returnurl = self.search_youtube(searchdict) - elif searchdict["query_type"] == 'custom': - returnurl = self.search_info_custom(searchdict) - else: - returnurl = self.search_info_google(searchdict) + if searchdict["query"].lower() in self.cache[searchdict["query_type"]]: + return self.cache[searchdict["query_type"]][searchdict["query"]] - if returnurl and searchdict["query_type"] != 'custom': + searchdict["searchurl"] = self.search_url_assemble(searchdict) + returnurl = self.search_handler(searchdict) + + if returnurl: self.cache[searchdict["query_type"]][searchdict["query"].lower()] = str(returnurl) return returnurl - def search_info_custom(self, searchdict): - lookfor = searchdict["searchquery"] + def search_handler(self, searchdict): try: - var = requests.get(r'' + searchdict["query_url"] + lookfor, headers=self.header) + var = requests.get(r'' + searchdict["searchurl"], headers=self.header) except Exception as e: var = e var = None @@ -85,47 +95,16 @@ class Search(): return None return var.url - def search_info_google(self, searchdict, retry=False): - lookfor = searchdict["searchquery"] - if not botconfig.SpiceBot_Search.search_api or retry: - try: - var = requests.get(r'http://www.google.com/search?q=' + lookfor + '&btnI', headers=self.header) - except Exception as e: - var = e - var = None - if not var or not var.url: - return None - return var.url - else: - return self.search_info_google(searchdict, retry=True) - - def search_maps_google(self, searchdict, retry=False): - lookfor = searchdict["searchquery"] - if not botconfig.SpiceBot_Search.search_api or retry: - try: - var = requests.get(r'http://www.google.com/maps/place/' + lookfor, headers=self.header) - except Exception as e: - var = e - var = None - if not var or not var.url: - return None - return var.url - else: - return self.search_maps_google(searchdict, retry=True) - - def search_youtube(self, searchdict, retry=False): - lookfor = searchdict["searchquery"] - if not botconfig.SpiceBot_Search.search_api or retry: - try: - var = requests.get(r'https://www.youtube.com/search?q=' + lookfor + '&btnI', headers=self.header) - except Exception as e: - var = e - var = None - if not var or not var.url: - return None - return var.url - else: - return self.search_youtube(searchdict, retry=True) + def search_url_assemble(self, searchdict): + # url base + url = str(searchdict) + # query + url += str(self.valid_api[searchdict["query_type"]]['query']) + str(searchdict["query"]) + # nsfw search? TODO + # additional parts + if "additional_url" in list(self.valid_api[searchdict["query_type"]].keys()): + url += str(self.valid_api[searchdict["query_type"]]['additional_url']) + return url search = Search() diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json b/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json deleted file mode 100644 index 1fc801a..0000000 --- a/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json +++ /dev/null @@ -1,8 +0,0 @@ -{ -"?default": { - "type": "search", - "responses": ["http://community.spiceworks.com/search?query="], - "blank_fail": ["https://www.spiceworks.com/"], - "search_fail": ["No Spiceworks forum posts found!"], - }, -} diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json new file mode 100644 index 0000000..34b84c3 --- /dev/null +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json @@ -0,0 +1,4 @@ +{ +"url": "http://www.google.com/maps/place/", +"query": '', +} diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/google.json b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/google.json new file mode 100644 index 0000000..f9f3bf5 --- /dev/null +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/google.json @@ -0,0 +1,5 @@ +{ +"url": "http://www.google.com/search?", +"query": 'q=', +"additional_url": "&btnI", +} diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/spiceworks.json b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/spiceworks.json new file mode 100644 index 0000000..31f9c1d --- /dev/null +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/spiceworks.json @@ -0,0 +1,4 @@ +{ +"url": "http://community.spiceworks.com/search?", +"query": 'query=', +} diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/youtube.json b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/youtube.json new file mode 100644 index 0000000..220eb26 --- /dev/null +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/youtube.json @@ -0,0 +1,5 @@ +{ +"url": "http://www.youtube.com/search?", +"query": 'q=', +"additional_url": "&btnI", +} diff --git a/sopel_modules/SpiceBot_Core_Commands/AI.py b/sopel_modules/SpiceBot_Core_Commands/AI.py index 3dbf8c8..1e7b928 100644 --- a/sopel_modules/SpiceBot_Core_Commands/AI.py +++ b/sopel_modules/SpiceBot_Core_Commands/AI.py @@ -176,7 +176,7 @@ def bot_command_rule_ai(bot, trigger): if searchterm: if trigger_args[0].lower() == "where": searchdict = { - "type": "maps", + "type": "gmaps", "query": searchterm, } searchreturn = SpiceBot.search.search(searchdict) diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index dbad838..9a5e9e3 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -298,12 +298,11 @@ def bot_dictcom_search(bot, trigger, botcom): botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = botcom.dict["dict"][botcom.dict["responsekey"]]["blank_fail"] return bot_dictcom_reply_shared(bot, trigger, botcom) - searchurl = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0] + query_type = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0] searchterm = botcom.dict["completestring"] searchdict = { - "query_type": "custom", - "query_url": searchurl, + "query_type": query_type, "query": searchterm, } diff --git a/sopel_modules/SpiceBot_Core_Tools/Search.py b/sopel_modules/SpiceBot_Core_Tools/Search.py index 42419ad..24f96e6 100644 --- a/sopel_modules/SpiceBot_Core_Tools/Search.py +++ b/sopel_modules/SpiceBot_Core_Tools/Search.py @@ -11,7 +11,7 @@ import sopel_modules.SpiceBot as SpiceBot @SpiceBot.prerun('module') -@sopel.module.commands('search', 'find', 'google', 'lookup') +@sopel.module.commands('search', 'find', 'googled', 'lookup') def search_main(bot, trigger, botcom): searchterm = spicemanip(botcom.dict['args'], 0) or None @@ -40,7 +40,7 @@ def search_where(bot, trigger, botcom): return searchdict = { "query": searchterm, - "query_type": "maps", + "query_type": "gmaps", } searchreturn = SpiceBot.search.search(searchdict) @@ -70,7 +70,7 @@ def search_what(bot, trigger, botcom): @SpiceBot.prerun('module') -@sopel.module.commands("youtube", "video") +@sopel.module.commands("youtubes", "video") def search_video(bot, trigger, botcom): searchterm = spicemanip(botcom.dict['args'], 0) or None From 54f66a7ecbe67a141e55bf714d6f42dd15bf879f Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 14:35:05 -0400 Subject: [PATCH 02/36] dires --- sopel_modules/SpiceBot/Search.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index 81d0634..5d117a8 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -24,6 +24,7 @@ class Search(): def __init__(self): self.header = {'User-Agent': str(UserAgent().chrome)} self.setup_search() + self.dir_to_scan = botread.get_config_dirs("SpiceBot_Search") valid_search_api_dict = botread.json_to_dict_simple(self.dir_to_scan, "Search API", "SpiceBot_Search") self.cache = {} From 107b18d76e9cac509da057e35140e8b1594b6698 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 14:36:34 -0400 Subject: [PATCH 03/36] self.valid_api = {} --- sopel_modules/SpiceBot/Search.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index 5d117a8..e871866 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -26,6 +26,7 @@ class Search(): self.setup_search() self.dir_to_scan = botread.get_config_dirs("SpiceBot_Search") valid_search_api_dict = botread.json_to_dict_simple(self.dir_to_scan, "Search API", "SpiceBot_Search") + self.valid_api = {} self.cache = {} From a88e008b0f064c4bda8e57b254e16e6bf05ae012 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 14:43:41 -0400 Subject: [PATCH 04/36] test --- sopel_modules/SpiceBot/Search.py | 2 +- sopel_modules/SpiceBot_Core_Tools/Search.py | 24 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index e871866..47388ff 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -40,7 +40,7 @@ class Search(): if apikey: self.valid_api[search_api]["apikey"] = apikey - self.valid_api[search_api]["comtype"] = "gif_prefix" + self.valid_api[search_api]["comtype"] = "search_prefix" self.valid_api[search_api]["validcoms"] = [self.valid_api[search_api]["filename"]] botcommands.register(self.valid_api[search_api]) diff --git a/sopel_modules/SpiceBot_Core_Tools/Search.py b/sopel_modules/SpiceBot_Core_Tools/Search.py index 24f96e6..5a595be 100644 --- a/sopel_modules/SpiceBot_Core_Tools/Search.py +++ b/sopel_modules/SpiceBot_Core_Tools/Search.py @@ -30,6 +30,30 @@ def search_main(bot, trigger, botcom): bot.osd(["[Information search for '" + str(searchterm) + "']", str(searchreturn)]) +@SpiceBot.prerun('module', "search_prefix") +@sopel.module.commands('(.*)') +def gifapi_triggers(bot, trigger, botcom): + + if botcom.dict['com'] not in list(SpiceBot.search.valid_api.keys()): + return + + searchterm = spicemanip(botcom.dict['args'], 0) or None + if not searchterm: + SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "Not sure what you want me to look for.") + return + + searchdict = { + "query": searchterm, + "query_type": botcom.dict["realcom"], + } + + searchreturn = SpiceBot.search.search(searchdict) + if not searchreturn: + SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], 'I cannot find anything about that') + else: + bot.osd(["[" + botcom.dict["realcom"].upper() + " search for '" + str(searchterm) + "']", str(searchreturn)]) + + @SpiceBot.prerun('module') @sopel.module.commands("where", "whereis") def search_where(bot, trigger, botcom): From f27ac05826a9ee45a677a22bf1f5e632d22a6d76 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 14:44:59 -0400 Subject: [PATCH 05/36] apikey --- sopel_modules/SpiceBot/Search.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index 47388ff..3fea136 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -36,10 +36,6 @@ class Search(): self.valid_api[search_api]["apikey"] = None self.valid_api[search_api]["cache"] = dict() - apikey = eval("botconfig." + search_api + ".apikey") - if apikey: - self.valid_api[search_api]["apikey"] = apikey - self.valid_api[search_api]["comtype"] = "search_prefix" self.valid_api[search_api]["validcoms"] = [self.valid_api[search_api]["filename"]] botcommands.register(self.valid_api[search_api]) From cb8613d685be613cef43c97304c9269ad5b210b0 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 14:48:05 -0400 Subject: [PATCH 06/36] test --- sopel_modules/SpiceBot/Prerun.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sopel_modules/SpiceBot/Prerun.py b/sopel_modules/SpiceBot/Prerun.py index d299f97..77f8c35 100644 --- a/sopel_modules/SpiceBot/Prerun.py +++ b/sopel_modules/SpiceBot/Prerun.py @@ -462,6 +462,9 @@ def updatable_content(bot, botcom, trigger): def special_handling(botcom): botcom.dict["responsekey"] = "?default" + if "nonstockoptions" not in list(botcom.dict["dict"].keys()): + botcom.dict["dict"]["nonstockoptions"] = [] + # handling for special cases posscom = spicemanip(botcom.dict['args'], 1) if posscom.lower() in [command.lower() for command in botcom.dict["dict"]["nonstockoptions"]]: From e3a3c9773cef6e7265e82d63c02b9a71edb6332b Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 15:11:45 -0400 Subject: [PATCH 07/36] update read --- sopel_modules/SpiceBot/Read.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot/Read.py b/sopel_modules/SpiceBot/Read.py index 8014766..2fda138 100644 --- a/sopel_modules/SpiceBot/Read.py +++ b/sopel_modules/SpiceBot/Read.py @@ -283,13 +283,25 @@ class BotRead(): # check that type is set, use cases will inherit this if not set if "type" not in list(dict_from_file.keys()): - dict_from_file["type"] = 'module' + dict_from_file["type"] = dict_from_file["foldername"] dict_from_file["nonstockoptions"] = ["?default"] for command in list(dict_from_file.keys()): if command not in prerun_shared.stockoptions: dict_from_file["nonstockoptions"].append(command) + ignorelist = [] + if dict_from_file["type"] == "SpiceBot_Gif": + ignorelist.extend(["url", "query", "limit", "id", "api_id", "key", "nsfw", "sfw", "sfw", "results", "cururl", "additional_url"]) + elif dict_from_file["type"] == "SpiceBot_Search": + ignorelist.extend(["url", "query", "additional_url"]) + elif dict_from_file["type"] == "SpiceBot_Sherlock": + ignorelist.extend(["url", "errorType", "errorMsg", "noPeriod", "errorUrl"]) + + for ignoreitem in ignorelist: + if ignoreitem in dict_from_file["nonstockoptions"]: + dict_from_file["nonstockoptions"].remove(ignoreitem) + for specialcase in dict_from_file["nonstockoptions"]: # All of the above need to be in the dict if not From ad9c59f704813aac7cae4df379a2d65ae19eb8f6 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 15:12:29 -0400 Subject: [PATCH 08/36] test --- sopel_modules/SpiceBot/Gif.py | 2 +- sopel_modules/SpiceBot/Search.py | 2 +- sopel_modules/SpiceBot/Sherlock.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sopel_modules/SpiceBot/Gif.py b/sopel_modules/SpiceBot/Gif.py index ea9ff51..f87b7ab 100644 --- a/sopel_modules/SpiceBot/Gif.py +++ b/sopel_modules/SpiceBot/Gif.py @@ -36,7 +36,7 @@ class BotGif(): self.dir_to_scan = botread.get_config_dirs("SpiceBot_Gif") - valid_gif_api_dict = botread.json_to_dict_simple(self.dir_to_scan, "Gif API", "SpiceBot_Gif") + valid_gif_api_dict = botread.json_to_dict(self.dir_to_scan, "Gif API", "SpiceBot_Gif") """ 15:02 <+Exirel> `channel_section = getattr(bot.config, 'channel', None)` diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index 3fea136..ddcb8e8 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -25,7 +25,7 @@ class Search(): self.header = {'User-Agent': str(UserAgent().chrome)} self.setup_search() self.dir_to_scan = botread.get_config_dirs("SpiceBot_Search") - valid_search_api_dict = botread.json_to_dict_simple(self.dir_to_scan, "Search API", "SpiceBot_Search") + valid_search_api_dict = botread.json_to_dict(self.dir_to_scan, "Search API", "SpiceBot_Search") self.valid_api = {} self.cache = {} diff --git a/sopel_modules/SpiceBot/Sherlock.py b/sopel_modules/SpiceBot/Sherlock.py index 81af6aa..274ea93 100644 --- a/sopel_modules/SpiceBot/Sherlock.py +++ b/sopel_modules/SpiceBot/Sherlock.py @@ -25,7 +25,7 @@ class Sherlock(): dir_to_scan = botread.get_config_dirs("SpiceBot_Sherlock") - valid_usernames_dict = botread.json_to_dict_simple(dir_to_scan, "Sherlock", "SpiceBot_Sherlock") + valid_usernames_dict = botread.json_to_dict(dir_to_scan, "Sherlock", "SpiceBot_Sherlock") for social_network in list(valid_usernames_dict.keys()): self.dict[social_network] = valid_usernames_dict[social_network] From 5cfb88028e69bd8fe9431efdb1cf4c259446bfd7 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 15:17:58 -0400 Subject: [PATCH 09/36] test --- sopel_modules/SpiceBot/Read.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot/Read.py b/sopel_modules/SpiceBot/Read.py index 2fda138..ae487e0 100644 --- a/sopel_modules/SpiceBot/Read.py +++ b/sopel_modules/SpiceBot/Read.py @@ -240,7 +240,8 @@ class BotRead(): else: dict_from_file = {} - dict_from_file = self.command_defaults(dict_from_file) + # if len(list(dict_from_file.keys())): + # dict_from_file = self.command_defaults(dict_from_file) return dict_from_file def command_defaults(self, dict_from_file): From 1d85859124ab4f8d9013fcbaaf5d82286ecc3029 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 15:21:55 -0400 Subject: [PATCH 10/36] test --- sopel_modules/SpiceBot/Read.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sopel_modules/SpiceBot/Read.py b/sopel_modules/SpiceBot/Read.py index ae487e0..c69894c 100644 --- a/sopel_modules/SpiceBot/Read.py +++ b/sopel_modules/SpiceBot/Read.py @@ -246,6 +246,9 @@ class BotRead(): def command_defaults(self, dict_from_file): + if "?default" not in list(dict_from_file.keys()): + dict_from_file["?default"] = {} + # the command must have an author if "author" not in list(dict_from_file.keys()): dict_from_file["author"] = "deathbybandaid" From acf1f8ebe52e8fe3cfe3015e4f4208df1ad3d511 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 15:27:03 -0400 Subject: [PATCH 11/36] hmmmm --- sopel_modules/SpiceBot/Gif.py | 2 +- sopel_modules/SpiceBot/Search.py | 2 +- sopel_modules/SpiceBot/Sherlock.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sopel_modules/SpiceBot/Gif.py b/sopel_modules/SpiceBot/Gif.py index f87b7ab..ea9ff51 100644 --- a/sopel_modules/SpiceBot/Gif.py +++ b/sopel_modules/SpiceBot/Gif.py @@ -36,7 +36,7 @@ class BotGif(): self.dir_to_scan = botread.get_config_dirs("SpiceBot_Gif") - valid_gif_api_dict = botread.json_to_dict(self.dir_to_scan, "Gif API", "SpiceBot_Gif") + valid_gif_api_dict = botread.json_to_dict_simple(self.dir_to_scan, "Gif API", "SpiceBot_Gif") """ 15:02 <+Exirel> `channel_section = getattr(bot.config, 'channel', None)` diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index ddcb8e8..3fea136 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -25,7 +25,7 @@ class Search(): self.header = {'User-Agent': str(UserAgent().chrome)} self.setup_search() self.dir_to_scan = botread.get_config_dirs("SpiceBot_Search") - valid_search_api_dict = botread.json_to_dict(self.dir_to_scan, "Search API", "SpiceBot_Search") + valid_search_api_dict = botread.json_to_dict_simple(self.dir_to_scan, "Search API", "SpiceBot_Search") self.valid_api = {} self.cache = {} diff --git a/sopel_modules/SpiceBot/Sherlock.py b/sopel_modules/SpiceBot/Sherlock.py index 274ea93..81af6aa 100644 --- a/sopel_modules/SpiceBot/Sherlock.py +++ b/sopel_modules/SpiceBot/Sherlock.py @@ -25,7 +25,7 @@ class Sherlock(): dir_to_scan = botread.get_config_dirs("SpiceBot_Sherlock") - valid_usernames_dict = botread.json_to_dict(dir_to_scan, "Sherlock", "SpiceBot_Sherlock") + valid_usernames_dict = botread.json_to_dict_simple(dir_to_scan, "Sherlock", "SpiceBot_Sherlock") for social_network in list(valid_usernames_dict.keys()): self.dict[social_network] = valid_usernames_dict[social_network] From 70440a697bb6d83bd54213694c025fddf430c047 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 15:43:58 -0400 Subject: [PATCH 12/36] test --- sopel_modules/SpiceBot/Prerun.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sopel_modules/SpiceBot/Prerun.py b/sopel_modules/SpiceBot/Prerun.py index 77f8c35..e16f07a 100644 --- a/sopel_modules/SpiceBot/Prerun.py +++ b/sopel_modules/SpiceBot/Prerun.py @@ -439,6 +439,9 @@ def trigger_cant_run(bot, trigger, botcom, message=None): def updatable_content(bot, botcom, trigger): # commands that can be updated + if "updates_enabled" not in list(botcom.dict["dict"][botcom.dict["responsekey"]].keys()): + botcom.dict["dict"][botcom.dict["responsekey"]]["updates_enabled"] = False + if botcom.dict["dict"][botcom.dict["responsekey"]]["updates_enabled"]: customresponses = [] if botcom.dict["dict"][botcom.dict["responsekey"]]["updates_enabled"] == "shared": @@ -473,12 +476,24 @@ def special_handling(botcom): posscom = command botcom.dict["responsekey"] = posscom + botcom = dictkey_defaults(botcom) + if botcom.dict["responsekey"] != "?default": botcom.dict['args'] = spicemanip(botcom.dict['args'], '2+', 'list') return botcom +def dictkey_defaults(botcom): + if botcom.dict["responsekey"] not in list(botcom.dict["dict"].keys()): + botcom.dict["dict"][botcom.dict["responsekey"]] = {} + if "selection_allowed" not in list(botcom.dict["dict"][botcom.dict["responsekey"]].keys()): + botcom.dict["dict"][botcom.dict["responsekey"]]["selection_allowed"] = True + if "responses" not in list(botcom.dict["dict"][botcom.dict["responsekey"]].keys()): + botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = [] + return botcom + + def trigger_hyphen_args(botcom): hyphen_args = [] From bae0cd7bf6e14fba1338bccb955b0efd7b5694b5 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 15:45:51 -0400 Subject: [PATCH 13/36] test --- sopel_modules/SpiceBot/Prerun.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sopel_modules/SpiceBot/Prerun.py b/sopel_modules/SpiceBot/Prerun.py index e16f07a..7a5e5c7 100644 --- a/sopel_modules/SpiceBot/Prerun.py +++ b/sopel_modules/SpiceBot/Prerun.py @@ -485,6 +485,7 @@ def special_handling(botcom): def dictkey_defaults(botcom): + botcom.dict["dict"]["hardcoded_channel_block"] = [] if botcom.dict["responsekey"] not in list(botcom.dict["dict"].keys()): botcom.dict["dict"][botcom.dict["responsekey"]] = {} if "selection_allowed" not in list(botcom.dict["dict"][botcom.dict["responsekey"]].keys()): From 8344fc216d7f50f5856ab2a675a80cd2059979dd Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 15:50:55 -0400 Subject: [PATCH 14/36] test --- sopel_modules/SpiceBot_Core_Tools/Search.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sopel_modules/SpiceBot_Core_Tools/Search.py b/sopel_modules/SpiceBot_Core_Tools/Search.py index 5a595be..ff8c151 100644 --- a/sopel_modules/SpiceBot_Core_Tools/Search.py +++ b/sopel_modules/SpiceBot_Core_Tools/Search.py @@ -46,10 +46,11 @@ def gifapi_triggers(bot, trigger, botcom): "query": searchterm, "query_type": botcom.dict["realcom"], } + bot.say(str(searchdict)) searchreturn = SpiceBot.search.search(searchdict) if not searchreturn: - SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], 'I cannot find anything about that') + SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "[" + botcom.dict["realcom"].upper() + " search for '" + str(searchterm) + "']" + 'I cannot find anything about that') else: bot.osd(["[" + botcom.dict["realcom"].upper() + " search for '" + str(searchterm) + "']", str(searchreturn)]) @@ -69,7 +70,7 @@ def search_where(bot, trigger, botcom): searchreturn = SpiceBot.search.search(searchdict) if not searchreturn: - SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], 'I cannot find anything about that') + SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "[Location search for " + str(searchterm) + "] " + 'I cannot find anything about that') else: bot.osd(["[Location search for " + str(searchterm) + "]", str(searchreturn)]) @@ -88,7 +89,7 @@ def search_what(bot, trigger, botcom): searchreturn = SpiceBot.search.search(searchdict) if not searchreturn: - SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], 'I cannot find anything about that') + SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "[Information search for '" + str(searchterm) + "'] " + 'I cannot find anything about that') else: bot.osd(["[Information search for '" + str(searchterm) + "']", str(searchreturn)]) @@ -108,6 +109,6 @@ def search_video(bot, trigger, botcom): searchreturn = SpiceBot.search.search(searchdict) if not searchreturn: - SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], 'I cannot find anything about that') + SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "[Youtube search for '" + str(searchterm) + "'] " + 'I cannot find anything about that') else: bot.osd(["[Youtube search for '" + str(searchterm) + "']", str(searchreturn)]) From d163649b1ca4a53ffc6c5bd992cbcac94cb037e9 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 15:51:18 -0400 Subject: [PATCH 15/36] missed one --- sopel_modules/SpiceBot_Core_Tools/Search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot_Core_Tools/Search.py b/sopel_modules/SpiceBot_Core_Tools/Search.py index ff8c151..6cf8898 100644 --- a/sopel_modules/SpiceBot_Core_Tools/Search.py +++ b/sopel_modules/SpiceBot_Core_Tools/Search.py @@ -25,7 +25,7 @@ def search_main(bot, trigger, botcom): searchreturn = SpiceBot.search.search(searchdict) if not searchreturn: - SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], 'I cannot find anything about that') + SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "[Information search for '" + str(searchterm) + "'] " + 'I cannot find anything about that') else: bot.osd(["[Information search for '" + str(searchterm) + "']", str(searchreturn)]) From 51847e7ebb00c536875b8b25b1d1781e25e5d955 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 15:53:52 -0400 Subject: [PATCH 16/36] test --- sopel_modules/SpiceBot_Core_Commands/AI.py | 9 ++++----- sopel_modules/SpiceBot_Core_Commands/DictComs.py | 2 +- sopel_modules/SpiceBot_Core_Tools/Search.py | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/sopel_modules/SpiceBot_Core_Commands/AI.py b/sopel_modules/SpiceBot_Core_Commands/AI.py index 1e7b928..3ba0e6a 100644 --- a/sopel_modules/SpiceBot_Core_Commands/AI.py +++ b/sopel_modules/SpiceBot_Core_Commands/AI.py @@ -186,12 +186,11 @@ def bot_command_rule_ai(bot, trigger): } searchreturn = SpiceBot.search.search(searchdict) if not searchreturn: - bot.osd('I cannot find anything about that') + searchreturn = 'I cannot find anything about that' + if trigger_args[0].lower() == "where": + bot.osd(["[Location search for " + str(searchterm) + "]", str(searchreturn)]) else: - if trigger_args[0].lower() == "where": - bot.osd(["[Location search for " + str(searchterm) + "]", str(searchreturn)]) - else: - bot.osd(["[Information search for '" + str(searchterm) + "']", str(searchreturn)]) + bot.osd(["[Information search for '" + str(searchterm) + "']", str(searchreturn)]) return elif trigger_args[0].lower() in ["can", "have"] and trigger_args[1].lower() in ["you"] and trigger_args[2].lower() in ["see", "seen"]: diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index 9a5e9e3..a6e2d89 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -310,7 +310,7 @@ def bot_dictcom_search(bot, trigger, botcom): if not searchreturn: botcom.dict["success"] = False - failmessage = ['I cannot find anything about that'] + failmessage = ["[" + searchdict["query_type"].upper() + " search for '" + str(searchterm) + "']" + 'I cannot find anything about that'] if botcom.dict["dict"][botcom.dict["responsekey"]]["search_fail"]: failmessage = botcom.dict["dict"][botcom.dict["responsekey"]]["search_fail"] botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = failmessage diff --git a/sopel_modules/SpiceBot_Core_Tools/Search.py b/sopel_modules/SpiceBot_Core_Tools/Search.py index 6cf8898..eda041a 100644 --- a/sopel_modules/SpiceBot_Core_Tools/Search.py +++ b/sopel_modules/SpiceBot_Core_Tools/Search.py @@ -50,7 +50,7 @@ def gifapi_triggers(bot, trigger, botcom): searchreturn = SpiceBot.search.search(searchdict) if not searchreturn: - SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "[" + botcom.dict["realcom"].upper() + " search for '" + str(searchterm) + "']" + 'I cannot find anything about that') + SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "[" + botcom.dict["realcom"].upper() + " search for '" + str(searchterm) + "'] " + 'I cannot find anything about that') else: bot.osd(["[" + botcom.dict["realcom"].upper() + " search for '" + str(searchterm) + "']", str(searchreturn)]) From 41ad7543296313cfda491b008c2e1eb69f261026 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 15:57:53 -0400 Subject: [PATCH 17/36] st --- sopel_modules/SpiceBot_Core_Commands/DictComs.py | 4 ++-- sopel_modules/SpiceBot_Core_Tools/Search.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index a6e2d89..f0dba90 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -310,12 +310,12 @@ def bot_dictcom_search(bot, trigger, botcom): if not searchreturn: botcom.dict["success"] = False - failmessage = ["[" + searchdict["query_type"].upper() + " search for '" + str(searchterm) + "']" + 'I cannot find anything about that'] + failmessage = ["[" + searchdict["query_type"].title() + " search for '" + str(searchterm) + "']" + 'I cannot find anything about that'] if botcom.dict["dict"][botcom.dict["responsekey"]]["search_fail"]: failmessage = botcom.dict["dict"][botcom.dict["responsekey"]]["search_fail"] botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = failmessage else: - botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = ["[Search for '" + str(searchterm) + "'] " + str(searchreturn)] + botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = ["[" + searchdict["query_type"].title() + " search for '" + str(searchterm) + "'] " + str(searchreturn)] botcom.dict["specified"] = False bot_dictcom_reply_shared(bot, trigger, botcom) diff --git a/sopel_modules/SpiceBot_Core_Tools/Search.py b/sopel_modules/SpiceBot_Core_Tools/Search.py index eda041a..187d65a 100644 --- a/sopel_modules/SpiceBot_Core_Tools/Search.py +++ b/sopel_modules/SpiceBot_Core_Tools/Search.py @@ -50,9 +50,9 @@ def gifapi_triggers(bot, trigger, botcom): searchreturn = SpiceBot.search.search(searchdict) if not searchreturn: - SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "[" + botcom.dict["realcom"].upper() + " search for '" + str(searchterm) + "'] " + 'I cannot find anything about that') + SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "[" + botcom.dict["realcom"].title() + " search for '" + str(searchterm) + "'] " + 'I cannot find anything about that') else: - bot.osd(["[" + botcom.dict["realcom"].upper() + " search for '" + str(searchterm) + "']", str(searchreturn)]) + bot.osd(["[" + botcom.dict["realcom"].title() + " search for '" + str(searchterm) + "']", str(searchreturn)]) @SpiceBot.prerun('module') From edb80536721f55e3e1877c36a74694d4099d4122 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 16:01:45 -0400 Subject: [PATCH 18/36] test --- sopel_modules/SpiceBot/Search.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index 3fea136..df64bd6 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -12,6 +12,8 @@ from .Config import config as botconfig from .Read import read as botread from .Commands import commands as botcommands +from .Logs import logs + # TODO add a cache flush @@ -89,6 +91,7 @@ class Search(): except Exception as e: var = e var = None + logs.log('SpiceBot_Search', str(e)) if not var or not var.url: return None return var.url From aae6347cece3fe16be25dbd1bc9410db31222c60 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 16:03:26 -0400 Subject: [PATCH 19/36] test --- sopel_modules/SpiceBot/Search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index df64bd6..26a5d96 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -100,7 +100,7 @@ class Search(): # url base url = str(searchdict) # query - url += str(self.valid_api[searchdict["query_type"]]['query']) + str(searchdict["query"]) + url += str(self.valid_api[searchdict["query_type"]]) + str(searchdict["query"]) # nsfw search? TODO # additional parts if "additional_url" in list(self.valid_api[searchdict["query_type"]].keys()): From 271d945eee56f536d5f71616cb32c3d285862971 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 09:33:57 -0400 Subject: [PATCH 20/36] fix --- sopel_modules/SpiceBot/Search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index 26a5d96..7101c9b 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -98,7 +98,7 @@ class Search(): def search_url_assemble(self, searchdict): # url base - url = str(searchdict) + url = str(searchdict["searchurl"]) # query url += str(self.valid_api[searchdict["query_type"]]) + str(searchdict["query"]) # nsfw search? TODO From d2441d7c7df63adb26197b8a2799cf5f64a7b06d Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 09:39:59 -0400 Subject: [PATCH 21/36] remove logging --- sopel_modules/SpiceBot/Search.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index 7101c9b..c448057 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -12,8 +12,6 @@ from .Config import config as botconfig from .Read import read as botread from .Commands import commands as botcommands -from .Logs import logs - # TODO add a cache flush @@ -62,6 +60,9 @@ class Search(): for key in query_defaults: if key not in list(searchdict.keys()): searchdict[key] = query_defaults[key] + for key in list(self.valid_api[searchdict["query_type"]].keys()): + if key not in list(searchdict.keys()): + searchdict[key] = self.valid_api[searchdict["query_type"]][key] # Replace spaces in search query searchdict["searchquery"] = urllib.request.pathname2url(searchdict["query"]) @@ -91,14 +92,13 @@ class Search(): except Exception as e: var = e var = None - logs.log('SpiceBot_Search', str(e)) if not var or not var.url: return None return var.url def search_url_assemble(self, searchdict): # url base - url = str(searchdict["searchurl"]) + url = str(searchdict["url"]) # query url += str(self.valid_api[searchdict["query_type"]]) + str(searchdict["query"]) # nsfw search? TODO From 9afa434b179dc84837df35d6baf6a4838fd4d5c4 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 09:46:18 -0400 Subject: [PATCH 22/36] test --- sopel_modules/SpiceBot/Search.py | 5 ++++- sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json | 1 + sopel_modules/SpiceBot_Configs/SpiceBot_Search/google.json | 2 +- .../SpiceBot_Configs/SpiceBot_Search/spiceworks.json | 2 +- sopel_modules/SpiceBot_Configs/SpiceBot_Search/youtube.json | 2 +- 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index c448057..83a64f5 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -12,6 +12,8 @@ from .Config import config as botconfig from .Read import read as botread from .Commands import commands as botcommands +from .Logs import logs + # TODO add a cache flush @@ -100,11 +102,12 @@ class Search(): # url base url = str(searchdict["url"]) # query - url += str(self.valid_api[searchdict["query_type"]]) + str(searchdict["query"]) + url += str(self.valid_api[searchdict["query_param"]]) + str(searchdict["query"]) # nsfw search? TODO # additional parts if "additional_url" in list(self.valid_api[searchdict["query_type"]].keys()): url += str(self.valid_api[searchdict["query_type"]]['additional_url']) + logs.log('SpiceBot_Search', str(url)) return url diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json index 34b84c3..ef8e5c6 100644 --- a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json @@ -1,4 +1,5 @@ { "url": "http://www.google.com/maps/place/", "query": '', +"query_param": '', } diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/google.json b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/google.json index f9f3bf5..5d1b270 100644 --- a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/google.json +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/google.json @@ -1,5 +1,5 @@ { "url": "http://www.google.com/search?", -"query": 'q=', +"query_param": 'q=', "additional_url": "&btnI", } diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/spiceworks.json b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/spiceworks.json index 31f9c1d..b69aafc 100644 --- a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/spiceworks.json +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/spiceworks.json @@ -1,4 +1,4 @@ { "url": "http://community.spiceworks.com/search?", -"query": 'query=', +"query_param": 'query=', } diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/youtube.json b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/youtube.json index 220eb26..ff11913 100644 --- a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/youtube.json +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/youtube.json @@ -1,5 +1,5 @@ { "url": "http://www.youtube.com/search?", -"query": 'q=', +"query_param": 'q=', "additional_url": "&btnI", } From 5f75e7a81d1f15705560f94a93a1e3461910c6d6 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 09:48:31 -0400 Subject: [PATCH 23/36] test --- sopel_modules/SpiceBot/Search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index 83a64f5..f785782 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -102,7 +102,7 @@ class Search(): # url base url = str(searchdict["url"]) # query - url += str(self.valid_api[searchdict["query_param"]]) + str(searchdict["query"]) + url += str(self.valid_api[searchdict["query_type"]]["query_param"]) + str(searchdict["query"]) # nsfw search? TODO # additional parts if "additional_url" in list(self.valid_api[searchdict["query_type"]].keys()): From 8b46d2a5df1658f443a5471a09078b3a44189800 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 09:50:46 -0400 Subject: [PATCH 24/36] search fixed --- sopel_modules/SpiceBot/Search.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index f785782..ab8669d 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -12,8 +12,6 @@ from .Config import config as botconfig from .Read import read as botread from .Commands import commands as botcommands -from .Logs import logs - # TODO add a cache flush @@ -107,7 +105,6 @@ class Search(): # additional parts if "additional_url" in list(self.valid_api[searchdict["query_type"]].keys()): url += str(self.valid_api[searchdict["query_type"]]['additional_url']) - logs.log('SpiceBot_Search', str(url)) return url From 49e5b95d56e42263127960e6cab77cd9a2f35630 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 09:51:48 -0400 Subject: [PATCH 25/36] test --- sopel_modules/SpiceBot/Search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index f785782..56ff007 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -96,6 +96,7 @@ class Search(): var = None if not var or not var.url: return None + logs.log('SpiceBot_Search', str(var)) return var.url def search_url_assemble(self, searchdict): @@ -107,7 +108,6 @@ class Search(): # additional parts if "additional_url" in list(self.valid_api[searchdict["query_type"]].keys()): url += str(self.valid_api[searchdict["query_type"]]['additional_url']) - logs.log('SpiceBot_Search', str(url)) return url From 388b6f222dbe579516f06325d2f1e378bd6c3593 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 09:52:29 -0400 Subject: [PATCH 26/36] test --- sopel_modules/SpiceBot/Search.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index 4e74e9f..56ff007 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -12,6 +12,8 @@ from .Config import config as botconfig from .Read import read as botread from .Commands import commands as botcommands +from .Logs import logs + # TODO add a cache flush From 76ea57bb7cb905ba7ab1c509ee6e362afbb7ba5d Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 09:52:57 -0400 Subject: [PATCH 27/36] test --- sopel_modules/SpiceBot/Search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index 56ff007..c2489d0 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -90,13 +90,13 @@ class Search(): def search_handler(self, searchdict): try: - var = requests.get(r'' + searchdict["searchurl"], headers=self.header) + var = requests.get(searchdict["searchurl"], headers=self.header) except Exception as e: var = e var = None if not var or not var.url: return None - logs.log('SpiceBot_Search', str(var)) + logs.log('SpiceBot_Search', str(var.url)) return var.url def search_url_assemble(self, searchdict): From 342ac27599eee8afc2cf4ab8e9c65c0621788e33 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 09:54:40 -0400 Subject: [PATCH 28/36] test --- sopel_modules/SpiceBot_Core_Tools/Search.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sopel_modules/SpiceBot_Core_Tools/Search.py b/sopel_modules/SpiceBot_Core_Tools/Search.py index 187d65a..e9233a5 100644 --- a/sopel_modules/SpiceBot_Core_Tools/Search.py +++ b/sopel_modules/SpiceBot_Core_Tools/Search.py @@ -46,7 +46,6 @@ def gifapi_triggers(bot, trigger, botcom): "query": searchterm, "query_type": botcom.dict["realcom"], } - bot.say(str(searchdict)) searchreturn = SpiceBot.search.search(searchdict) if not searchreturn: From 936aa385df5a873e4ff4003f205fa7605dd05c3d Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 10:20:22 -0400 Subject: [PATCH 29/36] test --- sopel_modules/SpiceBot/Search.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index c2489d0..d4af351 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -89,6 +89,16 @@ class Search(): return returnurl def search_handler(self, searchdict): + try: + results = requests.get(searchdict["searchurl"], headers=self.header).json() + except Exception as e: + results = e + logs.log('SpiceBot_Search', str(e)) + return None + logs.log('SpiceBot_Search', str(results)) + return None + + def search_handler_old(self, searchdict): try: var = requests.get(searchdict["searchurl"], headers=self.header) except Exception as e: @@ -96,7 +106,6 @@ class Search(): var = None if not var or not var.url: return None - logs.log('SpiceBot_Search', str(var.url)) return var.url def search_url_assemble(self, searchdict): From fdd06256bfdd190399474af78858e4f1759118e1 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 10:24:21 -0400 Subject: [PATCH 30/36] test --- sopel_modules/SpiceBot/Search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index d4af351..aad7d57 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -90,7 +90,7 @@ class Search(): def search_handler(self, searchdict): try: - results = requests.get(searchdict["searchurl"], headers=self.header).json() + results = requests.get(searchdict["searchurl"], headers=self.header).text() except Exception as e: results = e logs.log('SpiceBot_Search', str(e)) From 9843fd55e31161ebb9e708e0b070e2422d7caf96 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 10:44:55 -0400 Subject: [PATCH 31/36] test urban --- sopel_modules/SpiceBot/Search.py | 49 +++++++++++-------- .../SpiceBot_Search/gmaps.json | 1 - .../SpiceBot_Search/urban.json | 4 ++ 3 files changed, 33 insertions(+), 21 deletions(-) create mode 100644 sopel_modules/SpiceBot_Configs/SpiceBot_Search/urban.json diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index aad7d57..9ca7b3c 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals, absolute_import, division, print_functi import requests import urllib from fake_useragent import UserAgent +import json from sopel.config.types import StaticSection, ValidatedAttribute @@ -12,8 +13,6 @@ from .Config import config as botconfig from .Read import read as botread from .Commands import commands as botcommands -from .Logs import logs - # TODO add a cache flush @@ -89,24 +88,10 @@ class Search(): return returnurl def search_handler(self, searchdict): - try: - results = requests.get(searchdict["searchurl"], headers=self.header).text() - except Exception as e: - results = e - logs.log('SpiceBot_Search', str(e)) - return None - logs.log('SpiceBot_Search', str(results)) - return None - - def search_handler_old(self, searchdict): - try: - var = requests.get(searchdict["searchurl"], headers=self.header) - except Exception as e: - var = e - var = None - if not var or not var.url: - return None - return var.url + if searchdict["query_type"] in ["urban"]: + return self.search_urban(searchdict) + else: + return self.search_url(searchdict) def search_url_assemble(self, searchdict): # url base @@ -119,5 +104,29 @@ class Search(): url += str(self.valid_api[searchdict["query_type"]]['additional_url']) return url + def search_url(self, searchdict): + try: + var = requests.get(searchdict["searchurl"], headers=self.header) + except Exception as e: + var = e + var = None + if not var or not var.url: + return None + return var.url + + def search_urban(self, searchdict): + """Search Urban Dictionary.""" + try: + data = requests.get(searchdict["searchurl"]) + data = json.loads(data.decode('utf-8')) + except Exception as e: + data = e + return None + if data['result_type'] == 'no_results': + return None + result = data['list'][0] + response = "{0} - {1}".format(result['definition'].strip()[:256], searchdict["searchurl"]) + return response + search = Search() diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json index ef8e5c6..6a75151 100644 --- a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/gmaps.json @@ -1,5 +1,4 @@ { "url": "http://www.google.com/maps/place/", -"query": '', "query_param": '', } diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_Search/urban.json b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/urban.json new file mode 100644 index 0000000..5640878 --- /dev/null +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_Search/urban.json @@ -0,0 +1,4 @@ +{ +"url": "http://api.urbandictionary.com/v0/define?", +"query_param": 'term=', +} From d7fb2bab9fd75bc47b6e450785cfd2e3fff69011 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 10:50:42 -0400 Subject: [PATCH 32/36] test --- sopel_modules/SpiceBot/Search.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index 9ca7b3c..abe29b8 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -13,6 +13,8 @@ from .Config import config as botconfig from .Read import read as botread from .Commands import commands as botcommands +from .Logs import logs + # TODO add a cache flush @@ -120,6 +122,7 @@ class Search(): data = requests.get(searchdict["searchurl"]) data = json.loads(data.decode('utf-8')) except Exception as e: + logs.log(str(e), True) data = e return None if data['result_type'] == 'no_results': From 171b0f9a5283c30331cf2e59178e6aeb016fb899 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 10:54:33 -0400 Subject: [PATCH 33/36] test --- sopel_modules/SpiceBot/Search.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index abe29b8..50c50b6 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -46,6 +46,7 @@ class Search(): # check cache if search_api not in list(self.cache.keys()): self.cache[search_api] = dict() + logs.log("SpiceBot_Search", "Found " + str(len(list(self.valid_api.keys()))) + " search APIs", True) def setup_search(self): botconfig.define_section("SpiceBot_Search", SpiceBot_Search_MainSection, validate=False) @@ -122,7 +123,7 @@ class Search(): data = requests.get(searchdict["searchurl"]) data = json.loads(data.decode('utf-8')) except Exception as e: - logs.log(str(e), True) + logs.log("SpiceBot_Search", str(e), True) data = e return None if data['result_type'] == 'no_results': From 3af17d26d3106fc6aa0a510d64a76a2fc098e5b7 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 11:08:40 -0400 Subject: [PATCH 34/36] test --- sopel_modules/SpiceBot/Search.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index 50c50b6..d26ef15 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals, absolute_import, division, print_functi import requests import urllib from fake_useragent import UserAgent -import json +import re from sopel.config.types import StaticSection, ValidatedAttribute @@ -13,8 +13,6 @@ from .Config import config as botconfig from .Read import read as botread from .Commands import commands as botcommands -from .Logs import logs - # TODO add a cache flush @@ -46,7 +44,6 @@ class Search(): # check cache if search_api not in list(self.cache.keys()): self.cache[search_api] = dict() - logs.log("SpiceBot_Search", "Found " + str(len(list(self.valid_api.keys()))) + " search APIs", True) def setup_search(self): botconfig.define_section("SpiceBot_Search", SpiceBot_Search_MainSection, validate=False) @@ -118,19 +115,15 @@ class Search(): return var.url def search_urban(self, searchdict): - """Search Urban Dictionary.""" - try: - data = requests.get(searchdict["searchurl"]) - data = json.loads(data.decode('utf-8')) - except Exception as e: - logs.log("SpiceBot_Search", str(e), True) - data = e - return None - if data['result_type'] == 'no_results': - return None - result = data['list'][0] - response = "{0} - {1}".format(result['definition'].strip()[:256], searchdict["searchurl"]) - return response + query = requests.get(searchdict["searchurl"]).json() + if query: + try: + _definition = query['list'][0]['definition'].strip().replace(' ', ' ') + definition = re.sub(r"\[*\]*", "", _definition) + return str(searchdict["query"]) + " " + str(definition) + except Exception as e: + definition = e + return None search = Search() From 9b8fd052aa0f7c8e5fa6c31f761541a026dfa1be Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 11:27:05 -0400 Subject: [PATCH 35/36] test --- sopel_modules/SpiceBot/Search.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index d26ef15..e9eced2 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -120,7 +120,8 @@ class Search(): try: _definition = query['list'][0]['definition'].strip().replace(' ', ' ') definition = re.sub(r"\[*\]*", "", _definition) - return str(searchdict["query"]) + " " + str(definition) + definition = re.findall('(\d\.[^\d]*)', definition) + return str(definition) except Exception as e: definition = e return None From 183bf4b637c0fb73e42d803f8e24d504c44427c9 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Mon, 23 Sep 2019 11:28:36 -0400 Subject: [PATCH 36/36] test --- sopel_modules/SpiceBot/Search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index e9eced2..6dfba82 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -121,7 +121,7 @@ class Search(): _definition = query['list'][0]['definition'].strip().replace(' ', ' ') definition = re.sub(r"\[*\]*", "", _definition) definition = re.findall('(\d\.[^\d]*)', definition) - return str(definition) + return str(definition[0]) except Exception as e: definition = e return None