From 7bbffecd869e91b676f60e63adacc34b57152657 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 10:09:11 -0400 Subject: [PATCH 01/15] test search dictcom --- .../SpiceBot_DictComs/search/testsearch.json | 7 +++++++ .../SpiceBot_Core_Commands/DictComs.py | 18 ++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/testsearch.json diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/testsearch.json b/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/testsearch.json new file mode 100644 index 0000000..977daa2 --- /dev/null +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/testsearch.json @@ -0,0 +1,7 @@ +{ +"?default": { + "type": "search", + "responses": "test", + "search_fail": ["Test search failed"], + }, +} diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index 67e07ed..5bc2d47 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -34,7 +34,7 @@ def bot_dictcom_process(bot, trigger, botcom): botcom.dict["target"] = False botcom.dict["success"] = True - if botcom.dict["dict"][botcom.dict["responsekey"]]["type"] in ['simple', 'fillintheblank', "target", 'targetplusreason', 'sayings', "readfromfile", "readfromurl", "ascii_art", "translate", "responses"]: + if botcom.dict["dict"][botcom.dict["responsekey"]]["type"] in ['simple', 'fillintheblank', "target", 'targetplusreason', 'sayings', "readfromfile", "readfromurl", "ascii_art", "translate", "responses", "search"]: return bot_dictcom_responses(bot, trigger, botcom) else: command_function_run = str('bot_dictcom_' + botcom.dict["dict"][botcom.dict["responsekey"]]["type"] + '(bot, trigger, botcom)') @@ -293,4 +293,18 @@ def bot_dictcom_feeds(bot, trigger, botcom): def bot_dictcom_search(bot, trigger, botcom): - bot.say("testing done") + + if botcom.dict["dict"][botcom.dict["responsekey"]]["blank_required"] and not botcom.dict["completestring"]: + botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = botcom.dict["dict"][botcom.dict["responsekey"]]["blank_fail"] + return bot_dictcom_reply_shared(bot, trigger, botcom) + elif botcom.dict["dict"][botcom.dict["responsekey"]]["blank_required"] and botcom.dict["completestring"]: + searchterm = [botcom.dict["completestring"]] + else: + searchterm = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] + + if searchterm: + searchreturn = SpiceBot.google.search(searchterm) + if not searchreturn: + bot.osd('I cannot find anything about that') + else: + bot.osd(["[Information search for '" + str(searchterm) + "']", str(searchreturn)]) From b7d5b38a6270910b23f6257169c54aef0dd550b5 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 10:18:58 -0400 Subject: [PATCH 02/15] better logic --- .../SpiceBot_Core_Commands/DictComs.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index 5bc2d47..7a5b2ad 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -302,9 +302,16 @@ def bot_dictcom_search(bot, trigger, botcom): else: searchterm = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] - if searchterm: - searchreturn = SpiceBot.google.search(searchterm) - if not searchreturn: - bot.osd('I cannot find anything about that') - else: - bot.osd(["[Information search for '" + str(searchterm) + "']", str(searchreturn)]) + searchreturn = SpiceBot.google.search(searchterm) + + if not searchreturn: + botcom.dict["success"] = False + falimessage = 'I cannot find anything about that' + if botcom.dict["dict"][botcom.dict["responsekey"]]["search_fail"]: + falimessage = botcom.dict["dict"][botcom.dict["responsekey"]]["search_fail"] + botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = falimessage + else: + botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = ["[Information search for '" + str(searchterm) + "']", str(searchreturn)] + + botcom.dict["specified"] = False + bot_dictcom_reply_shared(bot, trigger, botcom) From 94f9916302e3f7ce4f875c91bca0e92d3a846578 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 10:27:44 -0400 Subject: [PATCH 03/15] test --- sopel_modules/SpiceBot_Core_Commands/DictComs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index 7a5b2ad..8feb8aa 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -34,7 +34,7 @@ def bot_dictcom_process(bot, trigger, botcom): botcom.dict["target"] = False botcom.dict["success"] = True - if botcom.dict["dict"][botcom.dict["responsekey"]]["type"] in ['simple', 'fillintheblank', "target", 'targetplusreason', 'sayings', "readfromfile", "readfromurl", "ascii_art", "translate", "responses", "search"]: + if botcom.dict["dict"][botcom.dict["responsekey"]]["type"] in ['simple', 'fillintheblank', "target", 'targetplusreason', 'sayings', "readfromfile", "readfromurl", "ascii_art", "translate", "responses"]: return bot_dictcom_responses(bot, trigger, botcom) else: command_function_run = str('bot_dictcom_' + botcom.dict["dict"][botcom.dict["responsekey"]]["type"] + '(bot, trigger, botcom)') From abb72a9defa4eb0be14fb6153edab24a9a154e88 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 10:31:44 -0400 Subject: [PATCH 04/15] test --- sopel_modules/SpiceBot_Core_Commands/DictComs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index 8feb8aa..2c6a289 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -298,7 +298,7 @@ def bot_dictcom_search(bot, trigger, botcom): botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = botcom.dict["dict"][botcom.dict["responsekey"]]["blank_fail"] return bot_dictcom_reply_shared(bot, trigger, botcom) elif botcom.dict["dict"][botcom.dict["responsekey"]]["blank_required"] and botcom.dict["completestring"]: - searchterm = [botcom.dict["completestring"]] + searchterm = botcom.dict["completestring"] else: searchterm = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] @@ -306,10 +306,10 @@ def bot_dictcom_search(bot, trigger, botcom): if not searchreturn: botcom.dict["success"] = False - falimessage = 'I cannot find anything about that' + failmessage = ['I cannot find anything about that'] if botcom.dict["dict"][botcom.dict["responsekey"]]["search_fail"]: - falimessage = botcom.dict["dict"][botcom.dict["responsekey"]]["search_fail"] - botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = falimessage + failmessage = botcom.dict["dict"][botcom.dict["responsekey"]]["search_fail"] + botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = failmessage else: botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = ["[Information search for '" + str(searchterm) + "']", str(searchreturn)] From 203247e7fca4737a54dc11145048db84bbcffdfb Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 10:36:05 -0400 Subject: [PATCH 05/15] test --- sopel_modules/SpiceBot_Core_Commands/DictComs.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index 2c6a289..3114eab 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -298,11 +298,18 @@ def bot_dictcom_search(bot, trigger, botcom): botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = botcom.dict["dict"][botcom.dict["responsekey"]]["blank_fail"] return bot_dictcom_reply_shared(bot, trigger, botcom) elif botcom.dict["dict"][botcom.dict["responsekey"]]["blank_required"] and botcom.dict["completestring"]: - searchterm = botcom.dict["completestring"] + searchterm = [botcom.dict["completestring"]] else: searchterm = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] - searchreturn = SpiceBot.google.search(searchterm) + if botcom.dict["specified"]: + if botcom.dict["specified"] > len(searchterm): + botcom.dict["specified"] = len(searchterm) + query = spicemanip(searchterm, botcom.dict["specified"], 'return') + else: + query = spicemanip(searchterm, 'random', 'return') + + searchreturn = SpiceBot.google.search(query) if not searchreturn: botcom.dict["success"] = False From bb3c22ca0019ed3a22e52962384339f65a70af41 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 10:41:26 -0400 Subject: [PATCH 06/15] fix --- sopel_modules/SpiceBot_Core_Commands/DictComs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index 3114eab..21d365a 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -318,7 +318,7 @@ def bot_dictcom_search(bot, trigger, botcom): failmessage = botcom.dict["dict"][botcom.dict["responsekey"]]["search_fail"] botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = failmessage else: - botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = ["[Information search for '" + str(searchterm) + "']", str(searchreturn)] + botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = ["[Information search for '" + str(searchterm) + "'] " + str(searchreturn)] botcom.dict["specified"] = False bot_dictcom_reply_shared(bot, trigger, botcom) From 16f019b5b6c3d7a37361a07b828ebf62ad25ad02 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 10:44:55 -0400 Subject: [PATCH 07/15] test --- .../SpiceBot_DictComs/search/spiceworks.json | 7 +++++++ .../SpiceBot_DictComs/search/testsearch.json | 7 ------- sopel_modules/SpiceBot_Core_Commands/DictComs.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json delete mode 100644 sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/testsearch.json diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json b/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json new file mode 100644 index 0000000..cc80061 --- /dev/null +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json @@ -0,0 +1,7 @@ +{ +"?default": { + "type": "search", + "responses": "spiceworks", + "search_fail": ["No Spiceworks forum posts found!"], + }, +} diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/testsearch.json b/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/testsearch.json deleted file mode 100644 index 977daa2..0000000 --- a/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/testsearch.json +++ /dev/null @@ -1,7 +0,0 @@ -{ -"?default": { - "type": "search", - "responses": "test", - "search_fail": ["Test search failed"], - }, -} diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index 21d365a..32bfb1f 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -298,7 +298,7 @@ def bot_dictcom_search(bot, trigger, botcom): botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = botcom.dict["dict"][botcom.dict["responsekey"]]["blank_fail"] return bot_dictcom_reply_shared(bot, trigger, botcom) elif botcom.dict["dict"][botcom.dict["responsekey"]]["blank_required"] and botcom.dict["completestring"]: - searchterm = [botcom.dict["completestring"]] + searchterm = [botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] + " " + botcom.dict["completestring"]] else: searchterm = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] From 30d88f92fad7d642a690e1f189e6382a4ab02e05 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 10:46:55 -0400 Subject: [PATCH 08/15] test --- sopel_modules/SpiceBot_Core_Commands/DictComs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index 32bfb1f..fb98523 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -298,7 +298,7 @@ def bot_dictcom_search(bot, trigger, botcom): botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = botcom.dict["dict"][botcom.dict["responsekey"]]["blank_fail"] return bot_dictcom_reply_shared(bot, trigger, botcom) elif botcom.dict["dict"][botcom.dict["responsekey"]]["blank_required"] and botcom.dict["completestring"]: - searchterm = [botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] + " " + botcom.dict["completestring"]] + searchterm = [botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0] + " " + botcom.dict["completestring"]] else: searchterm = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] From 5ddef463315affdafb1a1d4a42cb39e9c74376c5 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 10:50:28 -0400 Subject: [PATCH 09/15] test --- .../SpiceBot_DictComs/search/spiceworks.json | 2 ++ sopel_modules/SpiceBot_Core_Commands/DictComs.py | 11 ++--------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json b/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json index cc80061..b4aec04 100644 --- a/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json @@ -2,6 +2,8 @@ "?default": { "type": "search", "responses": "spiceworks", + "blank_required": True, + "blank_fail": ["https://www.spiceworks.com/"], "search_fail": ["No Spiceworks forum posts found!"], }, } diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index fb98523..2c2dea8 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -300,16 +300,9 @@ def bot_dictcom_search(bot, trigger, botcom): elif botcom.dict["dict"][botcom.dict["responsekey"]]["blank_required"] and botcom.dict["completestring"]: searchterm = [botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0] + " " + botcom.dict["completestring"]] else: - searchterm = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] + searchterm = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0] - if botcom.dict["specified"]: - if botcom.dict["specified"] > len(searchterm): - botcom.dict["specified"] = len(searchterm) - query = spicemanip(searchterm, botcom.dict["specified"], 'return') - else: - query = spicemanip(searchterm, 'random', 'return') - - searchreturn = SpiceBot.google.search(query) + searchreturn = SpiceBot.google.search(searchterm) if not searchreturn: botcom.dict["success"] = False From 2f937f0732945e95ae311baa6316c9b12bc8b37c Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 10:53:47 -0400 Subject: [PATCH 10/15] create string representation --- sopel_modules/SpiceBot/Prerun.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sopel_modules/SpiceBot/Prerun.py b/sopel_modules/SpiceBot/Prerun.py index 3dfc1e9..d299f97 100644 --- a/sopel_modules/SpiceBot/Prerun.py +++ b/sopel_modules/SpiceBot/Prerun.py @@ -110,6 +110,8 @@ def prerun(t_command_type='module', t_command_subtype=None): args_pass, botcom = trigger_hyphen_arg_handler(bot, trigger, botcom) + botcom.dict["completestring"] = spicemanip(botcom.dict['args'], 0) + if args_pass: if trigger_runstatus(bot, trigger, botcom): From 8c99e08e4b9d6cf163d0dadff8579495e348f668 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 10:55:31 -0400 Subject: [PATCH 11/15] test --- sopel_modules/SpiceBot_Core_Commands/DictComs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index 2c2dea8..1b8dbc4 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -301,6 +301,7 @@ def bot_dictcom_search(bot, trigger, botcom): searchterm = [botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0] + " " + botcom.dict["completestring"]] else: searchterm = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0] + bot.say(str(searchterm)) searchreturn = SpiceBot.google.search(searchterm) From f3cc07cdca4ef208360eb0d36623c56f18c9b303 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 10:56:39 -0400 Subject: [PATCH 12/15] not a list --- sopel_modules/SpiceBot_Core_Commands/DictComs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index 1b8dbc4..3a235e4 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -298,10 +298,9 @@ def bot_dictcom_search(bot, trigger, botcom): botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = botcom.dict["dict"][botcom.dict["responsekey"]]["blank_fail"] return bot_dictcom_reply_shared(bot, trigger, botcom) elif botcom.dict["dict"][botcom.dict["responsekey"]]["blank_required"] and botcom.dict["completestring"]: - searchterm = [botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0] + " " + botcom.dict["completestring"]] + searchterm = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0] + " " + botcom.dict["completestring"] else: searchterm = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0] - bot.say(str(searchterm)) searchreturn = SpiceBot.google.search(searchterm) From 9b1b8d08320b20a985cb2700e4f1cd625f2cbf32 Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 11:50:41 -0400 Subject: [PATCH 13/15] test custom search --- sopel_modules/SpiceBot/Google.py | 105 -------------- sopel_modules/SpiceBot/Search.py | 131 ++++++++++++++++++ sopel_modules/SpiceBot/Sherlock.py | 2 +- sopel_modules/SpiceBot/__init__.py | 2 +- sopel_modules/SpiceBot/configure.py | 6 +- sopel_modules/SpiceBot/setup.py | 10 +- .../SpiceBot_DictComs/search/spiceworks.json | 3 +- sopel_modules/SpiceBot_Core_Commands/AI.py | 11 +- .../SpiceBot_Core_Commands/DictComs.py | 19 ++- sopel_modules/SpiceBot_Core_Tools/Search.py | 23 ++- 10 files changed, 182 insertions(+), 130 deletions(-) delete mode 100644 sopel_modules/SpiceBot/Google.py create mode 100644 sopel_modules/SpiceBot/Search.py diff --git a/sopel_modules/SpiceBot/Google.py b/sopel_modules/SpiceBot/Google.py deleted file mode 100644 index 044344b..0000000 --- a/sopel_modules/SpiceBot/Google.py +++ /dev/null @@ -1,105 +0,0 @@ -# coding=utf8 -from __future__ import unicode_literals, absolute_import, division, print_function -"""A way to search google""" - -import requests -from fake_useragent import UserAgent - -from sopel.config.types import StaticSection, ValidatedAttribute - -from .Config import config as botconfig - -# TODO add a cache flush - - -class SpiceBot_Google_MainSection(StaticSection): - search_api = ValidatedAttribute('search_api', default=None) - - -class Google(): - - def __init__(self): - self.header = {'User-Agent': str(UserAgent().chrome)} - self.setup_google() - self.cache = { - "info": {}, - "maps": {}, - "youtube": {} - } - - def setup_google(self): - botconfig.define_section("SpiceBot_Google", SpiceBot_Google_MainSection, validate=False) - - def search(self, searchterm, searchtype="info"): - - # losercase searching - searchterm = searchterm.lower() - - """ - safe='off', - """ - - # check cache - if searchtype not in list(self.cache.keys()): - self.cache[searchtype] = dict() - if searchterm in self.cache[searchtype]: - return self.cache[searchtype][searchterm] - - if searchtype == 'maps': - returnurl = self.search_maps(searchterm) - elif searchtype == 'youtube': - returnurl = self.search_youtube(searchterm) - else: - returnurl = self.search_info(searchterm) - - if returnurl: - self.cache[searchtype][searchterm] = str(returnurl) - return returnurl - - def search_info(self, searchterm, retry=False): - data = searchterm.replace(' ', '+') - lookfor = data.replace(':', '%3A') - if not botconfig.SpiceBot_Google.search_api or retry: - try: - var = requests.get(r'http://www.google.com/search?q=' + lookfor + '&btnI', headers=self.header) - except Exception as e: - var = e - var = None - if not var or not var.url: - return None - return var.url - else: - return self.search_info(searchterm, retry=True) - - def search_maps(self, searchterm, retry=False): - data = searchterm.replace(' ', '+') - lookfor = data.replace(':', '%3A') - if not botconfig.SpiceBot_Google.search_api or retry: - try: - var = requests.get(r'http://www.google.com/maps/place/' + lookfor, headers=self.header) - except Exception as e: - var = e - var = None - if not var or not var.url: - return None - return var.url - else: - return self.search_maps(searchterm, retry=True) - - def search_youtube(self, searchterm, retry=False): - data = searchterm.replace(' ', '+') - lookfor = data.replace(':', '%3A') - if not botconfig.SpiceBot_Google.search_api or retry: - try: - var = requests.get(r'https://www.youtube.com/search?q=' + lookfor + '&btnI', headers=self.header) - except Exception as e: - var = e - var = None - if not var or not var.url: - return None - return var.url - else: - return self.search_youtube(searchterm, retry=True) - - -google = Google() diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py new file mode 100644 index 0000000..c8006fb --- /dev/null +++ b/sopel_modules/SpiceBot/Search.py @@ -0,0 +1,131 @@ +# coding=utf8 +from __future__ import unicode_literals, absolute_import, division, print_function +"""A way to search on the internet""" + +import requests +import urllib +from fake_useragent import UserAgent + +from sopel.config.types import StaticSection, ValidatedAttribute + +from .Config import config as botconfig + +# TODO add a cache flush + + +class SpiceBot_Search_MainSection(StaticSection): + search_api = ValidatedAttribute('search_api', default=None) + + +class Search(): + + def __init__(self): + self.header = {'User-Agent': str(UserAgent().chrome)} + self.setup_search() + self.cache = { + "info": {}, + "maps": {}, + "youtube": {}, + "spiceworks": {}, + } + + def setup_search(self): + botconfig.define_section("SpiceBot_Search", SpiceBot_Search_MainSection, validate=False) + + def search(self, searchdict): + + # list of defaults + query_defaults = { + "query": None, + "query_type": "info", + "query_url": None + } + + # set defaults if they don't exist + for key in query_defaults: + if key not in list(searchdict.keys()): + searchdict[key] = query_defaults[key] + + # Replace spaces in search query + searchdict["searchquery"] = urllib.request.pathname2url(searchdict["query"]) + + """ + safe='off', + """ + + # check cache + if searchdict["query_type"] not in list(self.cache.keys()): + self.cache[searchdict["query_type"]] = dict() + if searchdict["query_type"] != "custom": + if searchdict["query"].lower() in self.cache[searchdict["query_type"]]: + return self.cache[searchdict["query_type"]][searchdict["query"]] + + if searchdict["query_type"] == 'maps': + returnurl = self.search_maps_google(searchdict) + elif searchdict["query_type"] == 'youtube': + returnurl = self.search_youtube(searchdict) + elif searchdict["query_type"] == 'custom': + returnurl = self.search_info_custom(searchdict) + else: + returnurl = self.search_info_google(searchdict) + + if returnurl and searchdict["query_type"] != 'custom': + self.cache[searchdict["query_type"]][searchdict["query"].lower()] = str(returnurl) + + return returnurl + + def search_info_custom(self, searchdict): + lookfor = searchdict["searchquery"] + try: + var = requests.get(r'' + searchdict["query_url"] + lookfor, headers=self.header) + except Exception as e: + var = e + var = None + if not var or not var.url: + return None + return var.url + + def search_info_google(self, searchdict, retry=False): + lookfor = searchdict["searchquery"] + if not botconfig.SpiceBot_Google.search_api or retry: + try: + var = requests.get(r'http://www.google.com/search?q=' + lookfor + '&btnI', headers=self.header) + except Exception as e: + var = e + var = None + if not var or not var.url: + return None + return var.url + else: + return self.search_info_google(searchdict, retry=True) + + def search_maps_google(self, searchdict, retry=False): + lookfor = searchdict["searchquery"] + if not botconfig.SpiceBot_Google.search_api or retry: + try: + var = requests.get(r'http://www.google.com/maps/place/' + lookfor, headers=self.header) + except Exception as e: + var = e + var = None + if not var or not var.url: + return None + return var.url + else: + return self.search_maps_google(searchdict, retry=True) + + def search_youtube(self, searchdict, retry=False): + lookfor = searchdict["searchquery"] + if not botconfig.SpiceBot_Google.search_api or retry: + try: + var = requests.get(r'https://www.youtube.com/search?q=' + lookfor + '&btnI', headers=self.header) + except Exception as e: + var = e + var = None + if not var or not var.url: + return None + return var.url + else: + return self.search_youtube(searchdict, retry=True) + + +search = Search() diff --git a/sopel_modules/SpiceBot/Sherlock.py b/sopel_modules/SpiceBot/Sherlock.py index 3c89bff..81af6aa 100644 --- a/sopel_modules/SpiceBot/Sherlock.py +++ b/sopel_modules/SpiceBot/Sherlock.py @@ -1,6 +1,6 @@ # coding=utf8 from __future__ import unicode_literals, absolute_import, division, print_function -"""A way to search google""" +"""A way to search for usernames""" from sopel.config.types import StaticSection, ListAttribute diff --git a/sopel_modules/SpiceBot/__init__.py b/sopel_modules/SpiceBot/__init__.py index 13a8f1f..f1da22e 100644 --- a/sopel_modules/SpiceBot/__init__.py +++ b/sopel_modules/SpiceBot/__init__.py @@ -30,7 +30,7 @@ from .Server import * from .Channels import * from .Users import * from .AI import * -from .Google import * +from .Search import * from .Gif import * from .Sherlock import * from .Reddit import * diff --git a/sopel_modules/SpiceBot/configure.py b/sopel_modules/SpiceBot/configure.py index 1358c01..b86e60f 100644 --- a/sopel_modules/SpiceBot/configure.py +++ b/sopel_modules/SpiceBot/configure.py @@ -4,7 +4,7 @@ from __future__ import unicode_literals, absolute_import, division, print_functi from .Channels import SpiceBot_Channels_MainSection from .Commands import SpiceBot_Commands_MainSection -from .Google import SpiceBot_Google_MainSection +from .Search import SpiceBot_Search_MainSection from .AI import SpiceBot_AI_MainSection from .Update import SpiceBot_Update_MainSection from .Gif import SpiceBot_Gif_MainSection @@ -47,8 +47,8 @@ def configure(config): config.define_section("SpiceBot_Commands", SpiceBot_Commands_MainSection, validate=False) config.SpiceBot_Commands.configure_setting('query_prefix', 'query prefix') - config.define_section("SpiceBot_Google", SpiceBot_Google_MainSection, validate=False) - config.SpiceBot_Google.configure_setting('search_api', 'api key for google searching') + config.define_section("SpiceBot_Search", SpiceBot_Search_MainSection, validate=False) + config.SpiceBot_Search.configure_setting('search_api', 'api key for google searching') config.define_section("SpiceBot_Gif", SpiceBot_Gif_MainSection, validate=False) config.SpiceBot_Gif.configure_setting('extra', 'SpiceBot_Gif API Extra directory') diff --git a/sopel_modules/SpiceBot/setup.py b/sopel_modules/SpiceBot/setup.py index d0f5896..fe8c5ff 100644 --- a/sopel_modules/SpiceBot/setup.py +++ b/sopel_modules/SpiceBot/setup.py @@ -13,7 +13,7 @@ from .Events import events as botevents from .Channels import SpiceBot_Channels_MainSection from .Commands import SpiceBot_Commands_MainSection -from .Google import SpiceBot_Google_MainSection +from .Search import SpiceBot_Search_MainSection from .Update import SpiceBot_Update_MainSection from .Reddit import SpiceBot_Reddit_MainSection from .osd import SopelWrapperOSD, ToolsOSD, SopelOSD, SpiceBot_OSD @@ -37,7 +37,7 @@ def setup(bot): setup_commands(bot) # Commands - setup_google(bot) + setup_search(bot) # OSD setup_osd(bot) @@ -94,9 +94,9 @@ def setup_commands(bot): bot.config.core.prefix_list.extend(bot.config.SpiceBot_Commands.query_prefix) -def setup_google(bot): - logs.log('SpiceBot_Google', "Setting Up Configuration") - bot.config.define_section("SpiceBot_Google", SpiceBot_Google_MainSection, validate=False) +def setup_search(bot): + logs.log('SpiceBot_Search', "Setting Up Configuration") + bot.config.define_section("SpiceBot_Search", SpiceBot_Search_MainSection, validate=False) def setup_startupmonologue(bot): diff --git a/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json b/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json index b4aec04..1fc801a 100644 --- a/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json +++ b/sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json @@ -1,8 +1,7 @@ { "?default": { "type": "search", - "responses": "spiceworks", - "blank_required": True, + "responses": ["http://community.spiceworks.com/search?query="], "blank_fail": ["https://www.spiceworks.com/"], "search_fail": ["No Spiceworks forum posts found!"], }, diff --git a/sopel_modules/SpiceBot_Core_Commands/AI.py b/sopel_modules/SpiceBot_Core_Commands/AI.py index 1f60084..3dbf8c8 100644 --- a/sopel_modules/SpiceBot_Core_Commands/AI.py +++ b/sopel_modules/SpiceBot_Core_Commands/AI.py @@ -175,9 +175,16 @@ def bot_command_rule_ai(bot, trigger): searchterm = spicemanip(trigger_args, "3+") or None if searchterm: if trigger_args[0].lower() == "where": - searchreturn = SpiceBot.google.search(searchterm, 'maps') + searchdict = { + "type": "maps", + "query": searchterm, + } + searchreturn = SpiceBot.search.search(searchdict) else: - searchreturn = SpiceBot.google.search(searchterm) + searchdict = { + "query": searchterm, + } + searchreturn = SpiceBot.search.search(searchdict) if not searchreturn: bot.osd('I cannot find anything about that') else: diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index 3a235e4..cb1354c 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -294,15 +294,20 @@ def bot_dictcom_feeds(bot, trigger, botcom): def bot_dictcom_search(bot, trigger, botcom): - if botcom.dict["dict"][botcom.dict["responsekey"]]["blank_required"] and not botcom.dict["completestring"]: + if not botcom.dict["completestring"]: botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = botcom.dict["dict"][botcom.dict["responsekey"]]["blank_fail"] return bot_dictcom_reply_shared(bot, trigger, botcom) - elif botcom.dict["dict"][botcom.dict["responsekey"]]["blank_required"] and botcom.dict["completestring"]: - searchterm = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0] + " " + botcom.dict["completestring"] - else: - searchterm = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0] - searchreturn = SpiceBot.google.search(searchterm) + searchurl = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0] + searchterm = botcom.dict["completestring"] + + searchdict = { + "type": "custom", + "query_url": searchurl, + "query": searchterm, + } + + searchreturn = SpiceBot.search.search(searchdict) if not searchreturn: botcom.dict["success"] = False @@ -311,7 +316,7 @@ def bot_dictcom_search(bot, trigger, botcom): failmessage = botcom.dict["dict"][botcom.dict["responsekey"]]["search_fail"] botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = failmessage else: - botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = ["[Information search for '" + str(searchterm) + "'] " + str(searchreturn)] + botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = ["[Search for '" + str(searchterm) + "'] " + str(searchreturn)] botcom.dict["specified"] = False bot_dictcom_reply_shared(bot, trigger, botcom) diff --git a/sopel_modules/SpiceBot_Core_Tools/Search.py b/sopel_modules/SpiceBot_Core_Tools/Search.py index 15d83a4..42419ad 100644 --- a/sopel_modules/SpiceBot_Core_Tools/Search.py +++ b/sopel_modules/SpiceBot_Core_Tools/Search.py @@ -19,7 +19,11 @@ def search_main(bot, trigger, botcom): SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "Not sure what you want me to look for.") return - searchreturn = SpiceBot.google.search(searchterm) + searchdict = { + "query": searchterm, + } + + searchreturn = SpiceBot.search.search(searchdict) if not searchreturn: SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], 'I cannot find anything about that') else: @@ -34,8 +38,12 @@ def search_where(bot, trigger, botcom): if not searchterm: SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "Not sure what you want me to look for.") return + searchdict = { + "query": searchterm, + "query_type": "maps", + } - searchreturn = SpiceBot.google.search(searchterm, 'maps') + searchreturn = SpiceBot.search.search(searchdict) if not searchreturn: SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], 'I cannot find anything about that') else: @@ -50,8 +58,11 @@ def search_what(bot, trigger, botcom): if not searchterm: SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "Not sure what you want me to look for.") return + searchdict = { + "query": searchterm, + } - searchreturn = SpiceBot.google.search(searchterm) + searchreturn = SpiceBot.search.search(searchdict) if not searchreturn: SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], 'I cannot find anything about that') else: @@ -66,8 +77,12 @@ def search_video(bot, trigger, botcom): if not searchterm: SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "Not sure what you want me to look for.") return + searchdict = { + "query": searchterm, + "query_type": "youtube", + } - searchreturn = SpiceBot.google.search(searchterm, 'youtube') + searchreturn = SpiceBot.search.search(searchdict) if not searchreturn: SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], 'I cannot find anything about that') else: From 8f5de7a73812676db7a6e319f64b816a59159dbc Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 11:53:18 -0400 Subject: [PATCH 14/15] SpiceBot_Search --- sopel_modules/SpiceBot/Search.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sopel_modules/SpiceBot/Search.py b/sopel_modules/SpiceBot/Search.py index c8006fb..7e51827 100644 --- a/sopel_modules/SpiceBot/Search.py +++ b/sopel_modules/SpiceBot/Search.py @@ -87,7 +87,7 @@ class Search(): def search_info_google(self, searchdict, retry=False): lookfor = searchdict["searchquery"] - if not botconfig.SpiceBot_Google.search_api or retry: + if not botconfig.SpiceBot_Search.search_api or retry: try: var = requests.get(r'http://www.google.com/search?q=' + lookfor + '&btnI', headers=self.header) except Exception as e: @@ -101,7 +101,7 @@ class Search(): def search_maps_google(self, searchdict, retry=False): lookfor = searchdict["searchquery"] - if not botconfig.SpiceBot_Google.search_api or retry: + if not botconfig.SpiceBot_Search.search_api or retry: try: var = requests.get(r'http://www.google.com/maps/place/' + lookfor, headers=self.header) except Exception as e: @@ -115,7 +115,7 @@ class Search(): def search_youtube(self, searchdict, retry=False): lookfor = searchdict["searchquery"] - if not botconfig.SpiceBot_Google.search_api or retry: + if not botconfig.SpiceBot_Search.search_api or retry: try: var = requests.get(r'https://www.youtube.com/search?q=' + lookfor + '&btnI', headers=self.header) except Exception as e: From 36f47cd49d5906feba5c8a0acb13896383be944e Mon Sep 17 00:00:00 2001 From: deathbybandaid Date: Fri, 20 Sep 2019 11:56:50 -0400 Subject: [PATCH 15/15] query_type --- sopel_modules/SpiceBot_Core_Commands/DictComs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sopel_modules/SpiceBot_Core_Commands/DictComs.py b/sopel_modules/SpiceBot_Core_Commands/DictComs.py index cb1354c..dbad838 100644 --- a/sopel_modules/SpiceBot_Core_Commands/DictComs.py +++ b/sopel_modules/SpiceBot_Core_Commands/DictComs.py @@ -302,7 +302,7 @@ def bot_dictcom_search(bot, trigger, botcom): searchterm = botcom.dict["completestring"] searchdict = { - "type": "custom", + "query_type": "custom", "query_url": searchurl, "query": searchterm, }