template实现完成

Hello.py

@@ -138,6 +138,16 @@ def crawlef_run():
     crawlef_start.start()
 
 
+def system_main():
+    from system.gui import system_main
+    system_main()
+
+
+def system_run():  # 不需要进度条
+    system_start = Process(target=system_main)
+    system_start.start()
+
+
 def cotan_main():
     global SCREEN
     SCREEN = DragWindow(alpha=0.97, width=1200, height=800)
@@ -423,6 +433,7 @@ def cotan_main():
         frame,
         text='系统扩展',
         cursor=button_cursor,
+        command=system_run,
         height=1,
         font=font3,
         bg=button_color,
@@ -465,4 +476,3 @@ def cotan_main():
 
 if __name__ == "__main__":
     cotan_main()
-

crawler/db.py

@@ -1,93 +0,0 @@
-import os
-import time
-
-from crawler.template import Database, AddDatabase, DatabaseControllerCustom, LogBase
-
-
-class CoTanDB(Database):
-    def __init__(self, name):
-        self.dir = rf"{os.getcwd()}/Database_dir/{name}.cotanDB"  # 创建DB文件
-        self.file = open(self.dir, "r+" if os.path.exists(self.dir) else "w+")
-        self.id = 0
-        self.name = name
-        for _ in self.file.readlines():
-            self.id += 1
-
-    def __str__(self):
-        return self.name
-
-    def close(self):
-        try:
-            self.file.close()
-        except BaseException:
-            pass
-
-    def add_new(self, data):
-        data_str = str(self.id)
-        for i in data:
-            data_str += "," + str(i)
-        data_str += "\n"
-        self.file.write(data_str)
-        self.file.flush()
-        self.id += 1
-
-    def remove(self):
-        self.file.close()
-        os.remove(self.dir)
-
-    def out_file(self, out_dir):
-        with open(out_dir + fr"/{self.name}.contanDB", "w") as f:
-            with open(self.dir) as g:
-                f.write(g.read())
-
-
-class DatabaseController(AddDatabase, DatabaseControllerCustom):  # data base控制器
-
-    def add_new(self, name, data):  # 添加新内容
-        database = self.database.get(name)
-        if database is None:
-            self.add_database(name)
-            database = self.database.get(name)
-        database.add_new(data)
-
-    def close(self, name):  # 关闭数据表
-        try:
-            self.database[name].close()
-            del self.database[name]
-        except BaseException:
-            pass
-
-    def close_all(self):  # 关闭所有数据表
-        for i in self.database:
-            self.database[i].close()
-        self.database = {}
-
-    def rm_database(self, name):  # 删除数据表
-        self.database[name].remove()
-        del self.database[name]
-
-    def out(self, name, save_dir):  # 输出数据表
-        self.database[name].out_file(save_dir)
-
-    def return_database(self):
-        return list(self.database.keys())
-
-
-class Log(LogBase):
-    def __init__(self, log_dir):
-        self.log_dir = log_dir
-        self.log_file = open(
-            log_dir + "/log.coTanLog",
-            "r+" if os.path.exists(log_dir + "log.coTanLog") else "w+",
-        )
-
-    def write(self, data):
-        self.log_file.write(
-            f"[{time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))}] "
-            + data
-            + "\n"
-        )
-        self.log_file.flush()
-
-    def close(self):
-        self.log_file.close()

crawler/template.py

@@ -1,5 +1,6 @@
 import bs4
 import hashlib
+import os
 import re as regular
 import threading
 import time
@@ -11,9 +12,6 @@ from time import sleep
 
 import requests
 
-from crawler import db
-from crawler.controller import Url, PageDownloader
-from crawler.db import CoTanDB
 from system import plugin_class_loading, get_path
 
 keys_name_dict = {
@@ -40,7 +38,6 @@ keys_name_dict = {
     "right": Keys.RIGHT,
     "left": Keys.LEFT,
 }  # 键-值映射
-data_base = db.DatabaseController()
 
 
 class Database(metaclass=ABCMeta):
@@ -107,6 +104,38 @@ class LogBase(metaclass=ABCMeta):
         pass
 
 
+class DatabaseController(AddDatabase, DatabaseControllerCustom):  # data base控制器
+
+    def add_new(self, name, data):  # 添加新内容
+        database = self.database.get(name)
+        if database is None:
+            self.add_database(name)
+            database = self.database.get(name)
+        database.add_new(data)
+
+    def close(self, name):  # 关闭数据表
+        try:
+            self.database[name].close()
+            del self.database[name]
+        except BaseException:
+            pass
+
+    def close_all(self):  # 关闭所有数据表
+        for i in self.database:
+            self.database[i].close()
+        self.database = {}
+
+    def rm_database(self, name):  # 删除数据表
+        self.database[name].remove()
+        del self.database[name]
+
+    def out(self, name, save_dir):  # 输出数据表
+        self.database[name].out_file(save_dir)
+
+    def return_database(self):
+        return list(self.database.keys())
+
+
 class PageBase:
     def __init__(self, time_out):
         self.url = ""
@@ -146,10 +175,10 @@ class Urlbase:
     url_count = 0  # url处理器个数
 
     def __init__(self, dic=f"", dic_run=f""):
-        Url.url_count += 1
+        Urlbase.url_count += 1
         self.save_dir = dic
-        dic += f"/url[{Url.url_count}].cot_url"
-        dic_run += f"/url_run[{Url.url_count}].cot_url"
+        dic += f"/url[{Urlbase.url_count}].cot_url"
+        dic_run += f"/url_run[{Urlbase.url_count}].cot_url"
         self.dir = dic
         self.dir_run = dic_run
         self.file = open(dic, "a")  # 写入url_history的文件
@@ -330,11 +359,11 @@ class RequestsBase(metaclass=ABCMeta):
 class PagedownloaderBase(SeleniumBase, RequestsBase, metaclass=ABCMeta):
     downloader_count = 0
 
-    def __init__(self, url: Url, dic=""):
+    def __init__(self, url, dic=""):
         self.url = url
         self.dir = dic
-        self.log = db.Log(dic)
-        PageDownloader.downloader_count += 1
+        self.log = Log(dic)
+        PagedownloaderBase.downloader_count += 1
         self.page_source_dict = {}  # 页面保存信息
         self.cookie_Thread = None  # 子进程
         self.browser = None
@@ -504,7 +533,7 @@ class PageDownloaderSelenium(PageDownloaderCookies):
 
 
 class PageParserBase:
-    def __init__(self, downloader: PageDownloader):
+    def __init__(self, downloader):
         self.downloader = downloader
         self.downloader.set_page_parser(self)
         self.func_list = []
@@ -1205,7 +1234,7 @@ class PageParserDataSource(PageParserFunc):
                         new_url = str(bs.string).replace("\n", "")
                     else:
                         new_url = bs.attrs.get(url_name, "")
-                    Url.add_url(new_url, **url_args)
+                    self.downloader.url.add_url(new_url, **url_args)
                 except BaseException:
                     pass
             update_func()  # 更新tkinter
@@ -1475,3 +1504,63 @@ class PageParserChains(PageParserChainsWindow, PageParserClick, PageParserChains
 
 for i in range(1, 13):  # F1 - F12按键
     keys_name_dict[f"f{i}"] = eval(f"Keys.F{i}")
+
+
+class CoTanDB(Database):
+    def __init__(self, name):
+        self.dir = rf"{os.getcwd()}/Database_dir/{name}.cotanDB"  # 创建DB文件
+        self.file = open(self.dir, "r+" if os.path.exists(self.dir) else "w+")
+        self.id = 0
+        self.name = name
+        for _ in self.file.readlines():
+            self.id += 1
+
+    def __str__(self):
+        return self.name
+
+    def close(self):
+        try:
+            self.file.close()
+        except BaseException:
+            pass
+
+    def add_new(self, data):
+        data_str = str(self.id)
+        for i in data:
+            data_str += "," + str(i)
+        data_str += "\n"
+        self.file.write(data_str)
+        self.file.flush()
+        self.id += 1
+
+    def remove(self):
+        self.file.close()
+        os.remove(self.dir)
+
+    def out_file(self, out_dir):
+        with open(out_dir + fr"/{self.name}.contanDB", "w") as f:
+            with open(self.dir) as g:
+                f.write(g.read())
+
+
+class Log(LogBase):
+    def __init__(self, log_dir):
+        self.log_dir = log_dir
+        self.log_file = open(
+            log_dir + "/log.coTanLog",
+            "r+" if os.path.exists(log_dir + "log.coTanLog") else "w+",
+        )
+
+    def write(self, data):
+        self.log_file.write(
+            f"[{time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))}] "
+            + data
+            + "\n"
+        )
+        self.log_file.flush()
+
+    def close(self):
+        self.log_file.close()
+
+
+data_base = DatabaseController()

machinelearning/gui.py

@@ -8,7 +8,6 @@ import webbrowser
 
 import machinelearning.controller
 import machinelearning.template
-from machinelearning import template
 
 calculation_list = []
 calculation_method = []

system/controller.py

@@ -0,0 +1,69 @@
+import os
+import shutil
+
+PATH = os.getcwd() + rf'{os.sep}template'
+
+
+class NamingError(BaseException):
+    pass
+
+
+class ConflictError(BaseException):
+    pass
+
+
+class Plugin:
+    def __init__(self):
+        self.dir_list = []
+        self.plugin_list = []
+
+    def get_dir(self):
+        self.dir_list = os.listdir(PATH)
+        return self.dir_list.copy()
+
+    def get_all_plugin(self):
+        self.get_dir()
+        self.plugin_list = []
+        for name in self.dir_list:
+            try:
+                plugin = os.listdir(f'{PATH}{os.sep}{name}')
+            except BaseException:
+                pass
+            else:
+                self.plugin_list += [f'{name}{os.sep}{i}' for i in plugin]
+        return self.plugin_list.copy()
+
+    def get_plugin(self, index):
+        dir_name = self.dir_list[index]
+        return [f'{dir_name}{os.sep}{i}' for i in os.listdir(f'{PATH}{os.sep}{dir_name}')]
+
+    def add_plugin(self, index, plugin_dir):
+        plugin_name = os.path.split(plugin_dir)[1]
+        if plugin_name.startswith('template_') and plugin_name.endswith('.py'):
+            dir_name = self.dir_list[index]
+            targets_name = f'{PATH}{os.sep}{dir_name}{os.sep}{plugin_name}'
+            if os.path.isfile(targets_name):
+                raise ConflictError
+            shutil.copyfile(plugin_dir, targets_name)
+            return self.get_all_plugin()
+        else:
+            print(plugin_name)
+            raise NamingError
+
+    def del_plugin(self, index):
+        targets_name = f'{PATH}{os.sep}{self.plugin_list[index]}'
+        os.remove(targets_name)
+        return self.get_all_plugin()
+
+    def merge_plugin(self, index, plugin_dir):
+        plugin_name = os.path.split(plugin_dir)[1]
+        targets_name = f'{PATH}{os.sep}{self.dir_list[index]}{os.sep}{plugin_name}'
+        name = plugin_name[9:-3]
+        with open(targets_name, 'a+') as f:
+            with open(plugin_dir, 'r') as g:
+                f.write('\n' + g.read().replace('base = None', f'base = {name}'))
+
+    def show_plugin(self, index):
+        with open(f'{PATH}{os.sep}{self.plugin_list[index]}') as f:
+            code = f.read() + '\n[END]'
+        return code, self.plugin_list[index]

system/gui.py

@@ -0,0 +1,138 @@
+import tkinter
+from tkinter.filedialog import askopenfilename
+from tkinter.messagebox import showwarning, askokcancel
+from tkinter.scrolledtext import ScrolledText
+
+from system.controller import Plugin, NamingError, ConflictError
+
+SCREEN = tkinter.Tk()
+plugin = Plugin()
+SCREEN.title('插件管理')
+SCREEN.resizable(width=False, height=False)
+SCREEN.geometry(f'+10+10')
+bg_color = "#FFFAFA"  # 主颜色
+SCREEN["bg"] = bg_color
+botton_color = "#FFFAFA"  # 按钮颜色
+word_color = "#000000"  # 文字颜色
+FONT = ("黑体", 11)  # 设置字体
+gui_width = 13  # 标准宽度
+gui_height = 2
+row = 0
+column = 0
+
+
+def code_window(name):
+    global bg_color
+    cli_screen = tkinter.Toplevel(bg=bg_color)
+    cli_screen.title(f'插件查看器:{name}')
+    cli_screen.geometry("+10+10")  # 设置所在位置
+    cli_screen.resizable(width=False, height=False)
+
+    class ScrolledCli(ScrolledText):
+        def __init__(self, *args, **kwargs):
+            super(ScrolledCli, self).__init__(*args, **kwargs)
+
+        def insert(self, index, chars, *args):
+            text.config(state=tkinter.NORMAL)
+            super(ScrolledCli, self).insert(index, chars, *args)
+            text.config(state=tkinter.DISABLED)
+
+        def clear(self):
+            text.config(state=tkinter.NORMAL)
+            self.delete("0.0", tkinter.END)
+            text.config(state=tkinter.DISABLED)
+
+    text = ScrolledCli(cli_screen, font=("黑体", 11), height=30, width=100)
+    text.grid(column=0, row=0, columnspan=5, sticky=tkinter.E + tkinter.W)
+    text.config(state=tkinter.DISABLED)
+    cli_screen.update()
+    return text, cli_screen
+
+
+def get_dir():
+    plugin_dir_box.delete(0, tkinter.END)
+    plugin_dir_box.insert(0, *plugin.get_dir())
+
+
+def get_all_plugin():
+    plugin_box.delete(0, tkinter.END)
+    plugin_box.insert(0, *plugin.get_all_plugin())
+
+
+def get_plugin():
+    try:
+        index = plugin_dir_box.curselection()[0]
+    except IndexError:
+        return False
+    plugin_box.delete(0, tkinter.END)
+    plugin_box.insert(0, *plugin.get_plugin(index))
+
+
+def add_plugin():
+    index = plugin_dir_box.curselection()[0]
+    plugin_dir = askopenfilename(title='选择插件文件', filetypes=[("Python", ".py")])
+    try:
+        plugin_list = plugin.add_plugin(index, plugin_dir)
+    except NamingError:
+        showwarning('文件错误', '插件命名错误，命名规则:\ntemplate_[类\\方法名].py')
+    except ConflictError:
+        if askokcancel('提示', f'已经存在插件，是否需要尝试合并插件?\n[合并失败将产生不可逆的后果]'):
+            plugin.merge_plugin(index, plugin_dir)
+    except BaseException:
+        showwarning('文件错误', '插件导入遇到了未知错误')
+    else:
+        plugin_box.delete(0, tkinter.END)
+        plugin_box.insert(0, *plugin_list)
+
+
+def del_plugin():
+    index = plugin_box.curselection()[0]
+    try:
+        plugin_list = plugin.del_plugin(index)
+    except BaseException:
+        pass
+    else:
+        plugin_box.delete(0, tkinter.END)
+        plugin_box.insert(0, *plugin_list)
+
+
+def show_plugin():
+    index = plugin_box.curselection()[0]
+    try:
+        code, name = plugin.show_plugin(index)
+    except BaseException:
+        pass
+    else:
+        code_window(name)[0].insert(tkinter.END, code)
+
+
+def system_main():
+    global SCREEN
+    SCREEN.mainloop()
+
+
+(tkinter.Label(SCREEN, text='【插件管理】', bg=bg_color, fg=word_color, font=FONT, width=gui_width*3, height=gui_height)
+    .grid(column=column, row=row, columnspan=3))
+row += 1
+(tkinter.Button(SCREEN, bg=botton_color, fg=word_color, command=get_dir, text='查看插件列表', font=FONT, width=gui_width, height=gui_height)
+ .grid(column=column, row=row, sticky=tkinter.E + tkinter.W))
+(tkinter.Button(SCREEN, bg=botton_color, fg=word_color, command=get_all_plugin, text='查看所有插件', font=FONT, width=gui_width, height=gui_height)
+ .grid(column=column + 1, row=row, sticky=tkinter.E + tkinter.W))
+(tkinter.Button(SCREEN, bg=botton_color, fg=word_color, command=get_plugin, text='查看仓库插件', font=FONT, width=gui_width, height=gui_height)
+ .grid(column=column + 2, row=row, sticky=tkinter.E + tkinter.W))
+row += 1
+plugin_box = tkinter.Listbox(SCREEN, width=gui_width * 3, height=gui_height * 5)
+plugin_box.grid(column=column, row=row, columnspan=3, rowspan=5, sticky=tkinter.E + tkinter.W + tkinter.S + tkinter.N)
+row += 5
+(tkinter.Label(SCREEN, text='【插件仓库】', bg=bg_color, fg=word_color, font=FONT, width=gui_width*3, height=gui_height)
+    .grid(column=column, row=row, columnspan=3))
+row += 1
+plugin_dir_box = tkinter.Listbox(SCREEN, width=gui_width * 3, height=gui_height * 5)
+plugin_dir_box.grid(column=column, row=row, columnspan=3, rowspan=5, sticky=tkinter.E + tkinter.W + tkinter.S + tkinter.N)
+row += 5
+(tkinter.Button(SCREEN, bg=botton_color, fg=word_color, command=add_plugin, text='新增插件', font=FONT, width=gui_width, height=gui_height)
+ .grid(column=column, row=row, sticky=tkinter.E + tkinter.W))
+(tkinter.Button(SCREEN, bg=botton_color, fg=word_color, command=del_plugin, text='删除插件', font=FONT, width=gui_width, height=gui_height)
+ .grid(column=column + 1, row=row, sticky=tkinter.E + tkinter.W))
+(tkinter.Button(SCREEN, bg=botton_color, fg=word_color, command=show_plugin, text='查看插件', font=FONT, width=gui_width, height=gui_height)
+ .grid(column=column + 2, row=row, sticky=tkinter.E + tkinter.W))