test custom search

sopel_modules/SpiceBot/Google.py

@@ -1,105 +0,0 @@
-# coding=utf8
-from __future__ import unicode_literals, absolute_import, division, print_function
-"""A way to search google"""
-
-import requests
-from fake_useragent import UserAgent
-
-from sopel.config.types import StaticSection, ValidatedAttribute
-
-from .Config import config as botconfig
-
-# TODO add a cache flush
-
-
-class SpiceBot_Google_MainSection(StaticSection):
-    search_api = ValidatedAttribute('search_api', default=None)
-
-
-class Google():
-
-    def __init__(self):
-        self.header = {'User-Agent': str(UserAgent().chrome)}
-        self.setup_google()
-        self.cache = {
-                    "info": {},
-                    "maps": {},
-                    "youtube": {}
-                    }
-
-    def setup_google(self):
-        botconfig.define_section("SpiceBot_Google", SpiceBot_Google_MainSection, validate=False)
-
-    def search(self, searchterm, searchtype="info"):
-
-        # losercase searching
-        searchterm = searchterm.lower()
-
-        """
-        safe='off',
-        """
-
-        # check cache
-        if searchtype not in list(self.cache.keys()):
-            self.cache[searchtype] = dict()
-        if searchterm in self.cache[searchtype]:
-            return self.cache[searchtype][searchterm]
-
-        if searchtype == 'maps':
-            returnurl = self.search_maps(searchterm)
-        elif searchtype == 'youtube':
-            returnurl = self.search_youtube(searchterm)
-        else:
-            returnurl = self.search_info(searchterm)
-
-        if returnurl:
-            self.cache[searchtype][searchterm] = str(returnurl)
-        return returnurl
-
-    def search_info(self, searchterm, retry=False):
-        data = searchterm.replace(' ', '+')
-        lookfor = data.replace(':', '%3A')
-        if not botconfig.SpiceBot_Google.search_api or retry:
-            try:
-                var = requests.get(r'http://www.google.com/search?q=' + lookfor + '&btnI', headers=self.header)
-            except Exception as e:
-                var = e
-                var = None
-            if not var or not var.url:
-                return None
-            return var.url
-        else:
-            return self.search_info(searchterm, retry=True)
-
-    def search_maps(self, searchterm, retry=False):
-        data = searchterm.replace(' ', '+')
-        lookfor = data.replace(':', '%3A')
-        if not botconfig.SpiceBot_Google.search_api or retry:
-            try:
-                var = requests.get(r'http://www.google.com/maps/place/' + lookfor, headers=self.header)
-            except Exception as e:
-                var = e
-                var = None
-            if not var or not var.url:
-                return None
-            return var.url
-        else:
-            return self.search_maps(searchterm, retry=True)
-
-    def search_youtube(self, searchterm, retry=False):
-        data = searchterm.replace(' ', '+')
-        lookfor = data.replace(':', '%3A')
-        if not botconfig.SpiceBot_Google.search_api or retry:
-            try:
-                var = requests.get(r'https://www.youtube.com/search?q=' + lookfor + '&btnI', headers=self.header)
-            except Exception as e:
-                var = e
-                var = None
-            if not var or not var.url:
-                return None
-            return var.url
-        else:
-            return self.search_youtube(searchterm, retry=True)
-
-
-google = Google()

sopel_modules/SpiceBot/Search.py

@@ -0,0 +1,131 @@
+# coding=utf8
+from __future__ import unicode_literals, absolute_import, division, print_function
+"""A way to search on the internet"""
+
+import requests
+import urllib
+from fake_useragent import UserAgent
+
+from sopel.config.types import StaticSection, ValidatedAttribute
+
+from .Config import config as botconfig
+
+# TODO add a cache flush
+
+
+class SpiceBot_Search_MainSection(StaticSection):
+    search_api = ValidatedAttribute('search_api', default=None)
+
+
+class Search():
+
+    def __init__(self):
+        self.header = {'User-Agent': str(UserAgent().chrome)}
+        self.setup_search()
+        self.cache = {
+                    "info": {},
+                    "maps": {},
+                    "youtube": {},
+                    "spiceworks": {},
+                    }
+
+    def setup_search(self):
+        botconfig.define_section("SpiceBot_Search", SpiceBot_Search_MainSection, validate=False)
+
+    def search(self, searchdict):
+
+        # list of defaults
+        query_defaults = {
+                        "query": None,
+                        "query_type": "info",
+                        "query_url": None
+                        }
+
+        # set defaults if they don't exist
+        for key in query_defaults:
+            if key not in list(searchdict.keys()):
+                searchdict[key] = query_defaults[key]
+
+        # Replace spaces in search query
+        searchdict["searchquery"] = urllib.request.pathname2url(searchdict["query"])
+
+        """
+        safe='off',
+        """
+
+        # check cache
+        if searchdict["query_type"] not in list(self.cache.keys()):
+            self.cache[searchdict["query_type"]] = dict()
+        if searchdict["query_type"] != "custom":
+            if searchdict["query"].lower() in self.cache[searchdict["query_type"]]:
+                return self.cache[searchdict["query_type"]][searchdict["query"]]
+
+        if searchdict["query_type"] == 'maps':
+            returnurl = self.search_maps_google(searchdict)
+        elif searchdict["query_type"] == 'youtube':
+            returnurl = self.search_youtube(searchdict)
+        elif searchdict["query_type"] == 'custom':
+            returnurl = self.search_info_custom(searchdict)
+        else:
+            returnurl = self.search_info_google(searchdict)
+
+        if returnurl and searchdict["query_type"] != 'custom':
+            self.cache[searchdict["query_type"]][searchdict["query"].lower()] = str(returnurl)
+
+        return returnurl
+
+    def search_info_custom(self, searchdict):
+        lookfor = searchdict["searchquery"]
+        try:
+            var = requests.get(r'' + searchdict["query_url"] + lookfor, headers=self.header)
+        except Exception as e:
+            var = e
+            var = None
+        if not var or not var.url:
+            return None
+        return var.url
+
+    def search_info_google(self, searchdict, retry=False):
+        lookfor = searchdict["searchquery"]
+        if not botconfig.SpiceBot_Google.search_api or retry:
+            try:
+                var = requests.get(r'http://www.google.com/search?q=' + lookfor + '&btnI', headers=self.header)
+            except Exception as e:
+                var = e
+                var = None
+            if not var or not var.url:
+                return None
+            return var.url
+        else:
+            return self.search_info_google(searchdict, retry=True)
+
+    def search_maps_google(self, searchdict, retry=False):
+        lookfor = searchdict["searchquery"]
+        if not botconfig.SpiceBot_Google.search_api or retry:
+            try:
+                var = requests.get(r'http://www.google.com/maps/place/' + lookfor, headers=self.header)
+            except Exception as e:
+                var = e
+                var = None
+            if not var or not var.url:
+                return None
+            return var.url
+        else:
+            return self.search_maps_google(searchdict, retry=True)
+
+    def search_youtube(self, searchdict, retry=False):
+        lookfor = searchdict["searchquery"]
+        if not botconfig.SpiceBot_Google.search_api or retry:
+            try:
+                var = requests.get(r'https://www.youtube.com/search?q=' + lookfor + '&btnI', headers=self.header)
+            except Exception as e:
+                var = e
+                var = None
+            if not var or not var.url:
+                return None
+            return var.url
+        else:
+            return self.search_youtube(searchdict, retry=True)
+
+
+search = Search()

sopel_modules/SpiceBot/Sherlock.py

@@ -1,6 +1,6 @@
 # coding=utf8
 from __future__ import unicode_literals, absolute_import, division, print_function
-"""A way to search google"""
+"""A way to search for usernames"""
 
 from sopel.config.types import StaticSection, ListAttribute
 

sopel_modules/SpiceBot/__init__.py

@@ -30,7 +30,7 @@ from .Server import *
 from .Channels import *
 from .Users import *
 from .AI import *
-from .Google import *
+from .Search import *
 from .Gif import *
 from .Sherlock import *
 from .Reddit import *

sopel_modules/SpiceBot/configure.py

@@ -4,7 +4,7 @@ from __future__ import unicode_literals, absolute_import, division, print_functi
 
 from .Channels import SpiceBot_Channels_MainSection
 from .Commands import SpiceBot_Commands_MainSection
-from .Google import SpiceBot_Google_MainSection
+from .Search import SpiceBot_Search_MainSection
 from .AI import SpiceBot_AI_MainSection
 from .Update import SpiceBot_Update_MainSection
 from .Gif import SpiceBot_Gif_MainSection
@@ -47,8 +47,8 @@ def configure(config):
     config.define_section("SpiceBot_Commands", SpiceBot_Commands_MainSection, validate=False)
     config.SpiceBot_Commands.configure_setting('query_prefix', 'query prefix')
 
-    config.define_section("SpiceBot_Google", SpiceBot_Google_MainSection, validate=False)
+    config.define_section("SpiceBot_Search", SpiceBot_Search_MainSection, validate=False)
-    config.SpiceBot_Google.configure_setting('search_api', 'api key for google searching')
+    config.SpiceBot_Search.configure_setting('search_api', 'api key for google searching')
 
     config.define_section("SpiceBot_Gif", SpiceBot_Gif_MainSection, validate=False)
     config.SpiceBot_Gif.configure_setting('extra', 'SpiceBot_Gif API Extra directory')

sopel_modules/SpiceBot/setup.py

@@ -13,7 +13,7 @@ from .Events import events as botevents
 
 from .Channels import SpiceBot_Channels_MainSection
 from .Commands import SpiceBot_Commands_MainSection
-from .Google import SpiceBot_Google_MainSection
+from .Search import SpiceBot_Search_MainSection
 from .Update import SpiceBot_Update_MainSection
 from .Reddit import SpiceBot_Reddit_MainSection
 from .osd import SopelWrapperOSD, ToolsOSD, SopelOSD, SpiceBot_OSD
@@ -37,7 +37,7 @@ def setup(bot):
     setup_commands(bot)
 
     # Commands
-    setup_google(bot)
+    setup_search(bot)
 
     # OSD
     setup_osd(bot)
@@ -94,9 +94,9 @@ def setup_commands(bot):
     bot.config.core.prefix_list.extend(bot.config.SpiceBot_Commands.query_prefix)
 
 
-def setup_google(bot):
+def setup_search(bot):
-    logs.log('SpiceBot_Google', "Setting Up Configuration")
+    logs.log('SpiceBot_Search', "Setting Up Configuration")
-    bot.config.define_section("SpiceBot_Google", SpiceBot_Google_MainSection, validate=False)
+    bot.config.define_section("SpiceBot_Search", SpiceBot_Search_MainSection, validate=False)
 
 
 def setup_startupmonologue(bot):

sopel_modules/SpiceBot_Configs/SpiceBot_DictComs/search/spiceworks.json

@@ -1,8 +1,7 @@
 {
 "?default": {
               "type": "search",
-              "responses": "spiceworks",
+              "responses": ["http://community.spiceworks.com/search?query="],
-              "blank_required": True,
               "blank_fail": ["https://www.spiceworks.com/"],
               "search_fail": ["No Spiceworks forum posts found!"],
               },

sopel_modules/SpiceBot_Core_Commands/AI.py

@@ -175,9 +175,16 @@ def bot_command_rule_ai(bot, trigger):
             searchterm = spicemanip(trigger_args, "3+") or None
             if searchterm:
                 if trigger_args[0].lower() == "where":
-                    searchreturn = SpiceBot.google.search(searchterm, 'maps')
+                    searchdict = {
+                                    "type": "maps",
+                                    "query": searchterm,
+                                    }
+                    searchreturn = SpiceBot.search.search(searchdict)
                 else:
-                    searchreturn = SpiceBot.google.search(searchterm)
+                    searchdict = {
+                                    "query": searchterm,
+                                    }
+                    searchreturn = SpiceBot.search.search(searchdict)
                 if not searchreturn:
                     bot.osd('I cannot find anything about that')
                 else:

sopel_modules/SpiceBot_Core_Commands/DictComs.py

@@ -294,15 +294,20 @@ def bot_dictcom_feeds(bot, trigger, botcom):
 
 def bot_dictcom_search(bot, trigger, botcom):
 
-    if botcom.dict["dict"][botcom.dict["responsekey"]]["blank_required"] and not botcom.dict["completestring"]:
+    if not botcom.dict["completestring"]:
         botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = botcom.dict["dict"][botcom.dict["responsekey"]]["blank_fail"]
         return bot_dictcom_reply_shared(bot, trigger, botcom)
-    elif botcom.dict["dict"][botcom.dict["responsekey"]]["blank_required"] and botcom.dict["completestring"]:
-        searchterm = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0] + " " + botcom.dict["completestring"]
-    else:
-        searchterm = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0]
 
-    searchreturn = SpiceBot.google.search(searchterm)
+    searchurl = botcom.dict["dict"][botcom.dict["responsekey"]]["responses"][0]
+    searchterm = botcom.dict["completestring"]
+
+    searchdict = {
+                    "type": "custom",
+                    "query_url": searchurl,
+                    "query": searchterm,
+                    }
+
+    searchreturn = SpiceBot.search.search(searchdict)
 
     if not searchreturn:
         botcom.dict["success"] = False
@@ -311,7 +316,7 @@ def bot_dictcom_search(bot, trigger, botcom):
             failmessage = botcom.dict["dict"][botcom.dict["responsekey"]]["search_fail"]
         botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = failmessage
     else:
-        botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = ["[Information search for '" + str(searchterm) + "']    " + str(searchreturn)]
+        botcom.dict["dict"][botcom.dict["responsekey"]]["responses"] = ["[Search for '" + str(searchterm) + "']    " + str(searchreturn)]
 
     botcom.dict["specified"] = False
     bot_dictcom_reply_shared(bot, trigger, botcom)

sopel_modules/SpiceBot_Core_Tools/Search.py

@@ -19,7 +19,11 @@ def search_main(bot, trigger, botcom):
         SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "Not sure what you want me to look for.")
         return
 
-    searchreturn = SpiceBot.google.search(searchterm)
+    searchdict = {
+                    "query": searchterm,
+                    }
+
+    searchreturn = SpiceBot.search.search(searchdict)
     if not searchreturn:
         SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], 'I cannot find anything about that')
     else:
@@ -34,8 +38,12 @@ def search_where(bot, trigger, botcom):
     if not searchterm:
         SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "Not sure what you want me to look for.")
         return
+    searchdict = {
+                    "query": searchterm,
+                    "query_type": "maps",
+                    }
 
-    searchreturn = SpiceBot.google.search(searchterm, 'maps')
+    searchreturn = SpiceBot.search.search(searchdict)
     if not searchreturn:
         SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], 'I cannot find anything about that')
     else:
@@ -50,8 +58,11 @@ def search_what(bot, trigger, botcom):
     if not searchterm:
         SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "Not sure what you want me to look for.")
         return
+    searchdict = {
+                    "query": searchterm,
+                    }
 
-    searchreturn = SpiceBot.google.search(searchterm)
+    searchreturn = SpiceBot.search.search(searchdict)
     if not searchreturn:
         SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], 'I cannot find anything about that')
     else:
@@ -66,8 +77,12 @@ def search_video(bot, trigger, botcom):
     if not searchterm:
         SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], "Not sure what you want me to look for.")
         return
+    searchdict = {
+                    "query": searchterm,
+                    "query_type": "youtube",
+                    }
 
-    searchreturn = SpiceBot.google.search(searchterm, 'youtube')
+    searchreturn = SpiceBot.search.search(searchdict)
     if not searchreturn:
         SpiceBot.messagelog.messagelog_error(botcom.dict["log_id"], 'I cannot find anything about that')
     else: