From db61f9f3c4902efa78967a986cdece18e7a016f3 Mon Sep 17 00:00:00 2001 From: c-my Date: Sun, 6 Jun 2021 20:09:33 +0800 Subject: [PATCH] feat: baidubaike utils --- util.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/util.py b/util.py index bd41b67..5c8034d 100644 --- a/util.py +++ b/util.py @@ -1,6 +1,22 @@ +import os + url_file = "data/en_wiki_urls.txt" -if __name__ == '__main__': + +def build_baidubaike_url_list(): + file_list = [] + files = os.listdir("data/baidubaike") + for file in files: + file_list.append(file) + with open("data/baidu_baike_urls.txt", "w", encoding="utf-8") as f: + for file in file_list: + with open("data/baidubaike/" + file, "r", encoding="utf-8") as g: + urls = g.readlines() + for url in urls: + f.write("https://baike.baidu.com/item/" + url) + + +def build_wiki_url_list(): line_list = [] with open(url_file, "r", encoding="utf-8") as f: line_list = f.readlines() @@ -12,3 +28,7 @@ if __name__ == '__main__': with open("t.txt", "w", encoding="utf-8") as f: for r in result: f.write(r) + + +if __name__ == '__main__': + build_baidubaike_url_list()