initiel commit

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/NewsSpiderPy.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 3.9 (NewsSpiderPy)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (NewsSpiderPy)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/NewsSpiderPy.iml" filepath="$PROJECT_DIR$/.idea/NewsSpiderPy.iml" />
+    </modules>
+  </component>
+</project>

main.py

@@ -0,0 +1,63 @@
+# This is a sample Python script.
+
+# Press Shift+F10 to execute it or replace it with your code.
+# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
+from typing import Protocol, List
+
+from pageextractor.ChinanewsExtractor import ChinanewsExtractor
+from pageextractor.HuanqiuExtractor import HuanqiuExtractor
+from pageextractor.IFengExtractor import IFengExtractor
+from pageextractor.NeteaseExtractor import NeteaseExtractor
+from pageextractor.SinaExtractor import SinaExtractor
+from pageextractor.SohuExtractor import SohuExtractor
+from pageextractor.TecentExtractor import TecentExtractor
+from pageextractor.XinhuaExtractor import XinhuaExtractor
+from pageextractor.YouthExtractor import YouthExtractor
+from pageextractor.ZhonghuaExtractor import ZhonghuaExtractor
+from pageextractor.util import get_extractor
+from searchpageextractor.SinaSearchPageExtractor import SinaSearchPageExtractor
+from searchpageextractor.SogouSearchPageExtractor import SogouSearchPageExtractor
+
+
+def print_hi(name):
+    # Use a breakpoint in the code line below to debug your script.
+    print(f'Hi, {name}')  # Press Ctrl+F8 to toggle the breakpoint.
+
+
+def test_page_extractor():
+    pe = get_extractor("https://www.chinanews.com/gn/2021/09-27/9574765.shtml")
+    n_chinanews = pe.get_news()
+    pe = get_extractor("https://world.huanqiu.com/article/44j5MGsP4Cv")
+    n_huanqiu = pe.get_news()
+    pe = get_extractor("https://news.ifeng.com/c/89sSRm9IoyG")
+    n_ifeng = pe.get_news()
+    pe = get_extractor("https://www.163.com/news/article/GM4BUFRI0001899O.html?clickfrom=w_yw")
+    n_netease = pe.get_news()
+    pe = get_extractor("https://news.sina.com.cn/c/2021-10-12/doc-iktzqtyu1002491.shtml")
+    n_Sina = pe.get_news()
+    pe = get_extractor(
+        "https://www.sohu.com/a/494447997_114731?code=9d318a16a855848297532ebce9face3d&spm=smpc.home.top-news1.10.1634037047160aqOPOtD&_f=index_cpc_5")
+    n_sohu = pe.get_news()
+    pe = get_extractor("https://new.qq.com/omn/20210927/20210927A05QTU00.html")
+    n_tecent = pe.get_news()
+    pe = get_extractor("http://www.news.cn/mil/2021-09/16/c_1211371426.htm")
+    n_xinhua = pe.get_news()
+    pe = get_extractor("https://news.youth.cn/jsxw/202109/t20210916_13222690.htm")
+    n_youth = pe.get_news()
+    print(n_youth)
+    pe = get_extractor("https://news.china.com/domestic/945/20210809/39853554.html")
+    n_zhonghua = pe.get_news()
+
+
+def test_search_extractor():
+    spe = SinaSearchPageExtractor()
+    spe.search_by_keyword("冲突", 10)
+
+
+# Press the green button in the gutter to run the script.
+if __name__ == '__main__':
+    spe = SogouSearchPageExtractor()
+    r = spe.search_by_keyword("冲突", 10)
+    print_hi('PyCharm')
+
+# See PyCharm help at https://www.jetbrains.com/help/pycharm/

model/news.py

@@ -0,0 +1,9 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class News:
+    title: str
+    content: str
+    source: str
+    time: str

pageextractor/ChinanewsExtractor.py

@@ -0,0 +1,38 @@
+from pageextractor.PageExtractor import PageExtractor
+
+
+class ChinanewsExtractor(PageExtractor):
+
+    def get_title(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.content>h1")
+        return t.text.strip() if t is not None else ""
+
+    def get_source(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.content>div.left-time>div.left-t")
+        if t is None:
+            return ""
+        left_text = t.text
+        left_split = left_text.split("来源：")
+        if len(left_split) <= 1:
+            return ""
+        source = left_split[1].split("参与互动")[0]
+        return source.strip()
+
+    def get_time(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.content>div.left-time>div.left-t")
+        if t is None:
+            return ""
+        left_text = t.text
+        return left_text.split("来源")[0].strip()
+
+    def get_content(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.content>div.left_zw")
+        return t.text.strip() if t is not None else ""

pageextractor/HuanqiuExtractor.py

@@ -0,0 +1,27 @@
+from pageextractor.PageExtractor import PageExtractor
+
+
+class HuanqiuExtractor(PageExtractor):
+    def get_title(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.t-container-title>h3")
+        return t.text.strip() if t is not None else ""
+
+    def get_source(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.metadata-info span.source")
+        return t.text.strip() if t is not None else ""
+
+    def get_time(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.metadata-info p.time")
+        return t.text.strip() if t is not None else ""
+
+    def get_content(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.b-container div.l-con")
+        return t.text.strip() if t is not None else ""

pageextractor/IFengExtractor.py

@@ -0,0 +1,35 @@
+from pageextractor.PageExtractor import PageExtractor
+
+
+class IFengExtractor(PageExtractor):
+    def get_title(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div[class^=artical] h1[class^=topic-]")
+        if t is None:
+            t = self._soup.select_one("#artical_topic")
+        return t.text.strip() if t is not None else ""
+
+    def get_source(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div[class^=artical] div[class^=info-] p[class^=time]>span[class^=publisher]")
+        if t is None:
+            t = self._soup.select_one("span[itemprop=publisher]")
+        return t.text.strip() if t is not None else ""
+
+    def get_time(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div[class^=artical] div[class^=info-] p[class^=time]>span")
+        if t is None:
+            t = self._soup.select_one("span[itemprop=datePublished]")
+        return t.text.strip() if t is not None else ""
+
+    def get_content(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div[class^=artical] div[class^=main_content-]")
+        if t is None:
+            t = self._soup.select_one("div #artical_real")
+        return t.text.strip() if t is not None else ""

pageextractor/NeteaseExtractor.py

@@ -0,0 +1,32 @@
+from pageextractor.PageExtractor import PageExtractor
+
+
+class NeteaseExtractor(PageExtractor):
+    def get_title(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("h1.post_title")
+        return t.text.strip() if t is not None else ""
+
+    def get_source(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.post_info")
+        if t is None:
+            return ""
+        t_split = t.text.split("来源")
+        if len(t_split) <= 1:
+            return ""
+        return t.text.split("来源")[1].strip() if t is not None else ""
+
+    def get_time(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.post_info")
+        return t.text.split("来源")[0].strip() if t is not None else ""
+
+    def get_content(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.post_content>div.post_body")
+        return t.text.strip() if t is not None else ""

pageextractor/PageExtractor.py

@@ -0,0 +1,60 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+import requests
+from bs4 import BeautifulSoup
+
+from model.news import News
+
+
+class PageExtractor(ABC):
+    request_headers = {"User-Agent":
+                           "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
+                           "Chrome/94.0.4606.61 Safari/537.36"}
+
+    def __init__(self, url: str):
+        self._soup = self.get_soup(url)
+
+    def get_soup(self, url: str):
+        r = requests.get(url, headers=self.request_headers)
+
+        # 防止乱码
+        # 原理：如果requests没有在header中推测出编码，则按照协议，使用ISO-8859-1编码，这就是容易出现乱码的情形
+        # 因此使用get_encodings_from_content方法，从响应中探测编码（如html head中的meta中）
+        if r.encoding == 'ISO-8859-1':
+            encodings = requests.utils.get_encodings_from_content(r.text)
+            if encodings:
+                encoding = encodings[0]
+            else:
+                encoding = r.apparent_encoding
+            encode_content = r.content.decode(encoding, 'ignore')  # .encode('utf-8', 'ignore')
+        else:
+            encode_content = r.text
+        soup = BeautifulSoup(encode_content, features="lxml")
+        # soup = BeautifulSoup(r.text, features="lxml")
+        return soup
+
+    def get_news(self) -> Optional[News]:
+        title = self.get_title()
+        content = self.get_content()
+        source = self.get_source()
+        time = self.get_time()
+        if title == "" and content == "" and source == "" and time == "":
+            return None
+        return News(title, content, source, time)
+
+    @abstractmethod
+    def get_title(self) -> str:
+        ...
+
+    @abstractmethod
+    def get_source(self) -> str:
+        ...
+
+    @abstractmethod
+    def get_time(self) -> str:
+        ...
+
+    @abstractmethod
+    def get_content(self) -> str:
+        ...

pageextractor/SinaExtractor.py

@@ -0,0 +1,27 @@
+from pageextractor.PageExtractor import PageExtractor
+
+
+class SinaExtractor(PageExtractor):
+    def get_title(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("h1.main-title")
+        return t.text.strip() if t is not None else ""
+
+    def get_source(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.date-source a.source")
+        return t.text.strip() if t is not None else ""
+
+    def get_time(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.date-source span.date")
+        return t.text.strip() if t is not None else ""
+
+    def get_content(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("#article")
+        return t.text.strip() if t is not None else ""

pageextractor/SohuExtractor.py

@@ -0,0 +1,33 @@
+from pageextractor.PageExtractor import PageExtractor
+
+
+class SohuExtractor(PageExtractor):
+    def get_title(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.text-title h1")
+        if t is None:
+            t = self._soup.select_one("h3.article-title")
+        return t.text.strip() if t is not None else ""
+
+    def get_source(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.article-info span[data-role='original-link']")
+        return t.text.strip() if t is not None else ""
+
+    def get_time(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.article-info span.time")
+        if t is None:
+            t = self._soup.select_one("p.article-info span.time")
+        return t.text.strip() if t is not None else ""
+
+    def get_content(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("article.article")
+        if t is None:
+            t = self._soup.select_one("article.article-text")
+        return t.text.strip() if t is not None else ""

pageextractor/TecentExtractor.py

@@ -0,0 +1,21 @@
+from pageextractor.PageExtractor import PageExtractor
+
+
+class TecentExtractor(PageExtractor):
+    def get_title(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.LEFT h1")
+        return t.text.strip() if t is not None else ""
+
+    def get_source(self) -> str:
+        return ""
+
+    def get_time(self) -> str:
+        return ""
+
+    def get_content(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.content-article")
+        return t.text.strip() if t is not None else ""

pageextractor/XinhuaExtractor.py

@@ -0,0 +1,27 @@
+from pageextractor.PageExtractor import PageExtractor
+
+
+class XinhuaExtractor(PageExtractor):
+    def get_title(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.head-line span.title")
+        return t.text.strip() if t is not None else ""
+
+    def get_source(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.source")
+        return t.text.strip() if t is not None else ""
+
+    def get_time(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.header-time")
+        return t.text.strip() if t is not None else ""
+
+    def get_content(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("#detail")
+        return t.text.strip() if t is not None else ""

pageextractor/YouthExtractor.py

@@ -0,0 +1,27 @@
+from pageextractor.PageExtractor import PageExtractor
+
+
+class YouthExtractor(PageExtractor):
+    def get_title(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.page_title>h1")
+        return t.text.strip() if t is not None else ""
+
+    def get_source(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("#source_baidu")
+        return t.text.strip() if t is not None else ""
+
+    def get_time(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("#page_right")
+        return t.text.strip() if t is not None else ""
+
+    def get_content(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("div.page_left>div.page_k2")
+        return t.text.strip() if t is not None else ""

pageextractor/ZhonghuaExtractor.py

@@ -0,0 +1,27 @@
+from pageextractor.PageExtractor import PageExtractor
+
+
+class ZhonghuaExtractor(PageExtractor):
+    def get_title(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("h1.article_title")
+        return t.text.strip() if t is not None else ""
+
+    def get_source(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("span.source")
+        return t.text.strip() if t is not None else ""
+
+    def get_time(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("span.time")
+        return t.text.strip() if t is not None else ""
+
+    def get_content(self) -> str:
+        if self._soup is None:
+            return ""
+        t = self._soup.select_one("#js_article_content")
+        return t.text.strip() if t is not None else ""

pageextractor/util.py

@@ -0,0 +1,43 @@
+import re
+
+from pageextractor import PageExtractor
+from pageextractor.ChinanewsExtractor import ChinanewsExtractor
+from pageextractor.HuanqiuExtractor import HuanqiuExtractor
+from pageextractor.IFengExtractor import IFengExtractor
+from pageextractor.NeteaseExtractor import NeteaseExtractor
+from pageextractor.SinaExtractor import SinaExtractor
+from pageextractor.SohuExtractor import SohuExtractor
+from pageextractor.TecentExtractor import TecentExtractor
+from pageextractor.XinhuaExtractor import XinhuaExtractor
+from pageextractor.YouthExtractor import YouthExtractor
+from pageextractor.ZhonghuaExtractor import ZhonghuaExtractor
+
+
+def get_extractor(url: str) -> PageExtractor:
+    pattern = re.compile("://(.*?)/")
+    result = re.search(pattern, url)
+    if result is None:
+        return None
+    domain = result.group(1)
+    if domain == "www.sohu.com":
+        return SohuExtractor(url)
+    elif domain in ["www.163.com", "dy.163.com", "news.163.com"]:
+        return NeteaseExtractor(url)
+    elif domain == "new.qq.com":
+        return TecentExtractor(url)
+    elif domain in ["news.ifeng.com", "i.ifeng.com", "mil.ifeng.com"]:
+        return IFengExtractor(url)
+    elif domain == "www.chinanews.com":
+        return ChinanewsExtractor(url)
+    elif domain == "news.youth.cn":
+        return YouthExtractor(url)
+    elif domain in ["www.news.cn", "xinhuanet.com", "www.xinhuanet.com"]:
+        return XinhuaExtractor(url)
+    elif domain in ["world.huanqiu.com", "mil.huanqiu.com", "china.huanqiu.com"]:
+        return HuanqiuExtractor(url)
+    elif domain == "news.china.com":
+        return ZhonghuaExtractor(url)
+    elif domain == "news.sina.com.cn":
+        return SinaExtractor(url)
+    else:
+        return None

searchpageextractor/SearchPageExtractor.py

@@ -0,0 +1,10 @@
+from abc import ABC, abstractmethod
+from typing import List
+
+from model.news import News
+
+
+class SearchPageExtractor(ABC):
+    @abstractmethod
+    def search_by_keyword(self, keyword: str, max_page: int) -> List[News]:
+        ...

searchpageextractor/SinaSearchPageExtractor.py

@@ -0,0 +1,86 @@
+import concurrent.futures
+import re
+from typing import List, Optional
+
+import requests
+from bs4 import BeautifulSoup
+
+from model.news import News
+from pageextractor.SinaExtractor import SinaExtractor
+from searchpageextractor.SearchPageExtractor import SearchPageExtractor
+
+
+class SinaSearchPageExtractor(SearchPageExtractor):
+    _search_base_url = "https://search.sina.com.cn/news?q={}&c=news&sort=time"
+    _request_headers = {"User-Agent":
+                           "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
+                           "Chrome/94.0.4606.61 Safari/537.36"}
+
+    def search_by_keyword(self, keyword: str, max_page: int) -> List[News]:
+        all_item_links = self._get_all_item_links(keyword, max_page)
+        search_result = []
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+
+            futures = [executor.submit(SinaExtractor(link).get_news) for link in all_item_links]
+            for future in concurrent.futures.as_completed(futures):
+                r = future.result()
+                if r is not None:
+                    search_result.append(r)
+        return search_result
+
+    def _get_all_item_links(self, keyword: str, max_page: int) -> List[str]:
+        url_list = []
+        current_page = 1
+        search_page = self._get_search_page(keyword)
+        item_urls = self._get_item_urls(search_page)
+        url_list.extend(item_urls)
+        next_page_url = self._get_next_page_url(search_page)
+
+        while next_page_url is not None and current_page < max_page:
+            search_page = self._get_search_page_by_url(next_page_url)
+            item_urls = self._get_item_urls(search_page)
+            url_list.extend(item_urls)
+            next_page_url = self._get_next_page_url(search_page)
+            current_page += 1
+        return url_list
+
+    def _get_search_page(self, keyword: str) -> BeautifulSoup:
+        search_url = self._get_search_url(keyword)
+        return self._get_search_page_by_url(search_url)
+
+    def _get_item_urls(self, soup: BeautifulSoup) -> List[str]:
+        selector = "div.box-result h2>a"
+        item_links = soup.select(selector)
+        result_links = [link["href"] for link in item_links]
+        return result_links
+
+    def _get_search_url(self, keyword: str) -> str:
+        return self._search_base_url.format(keyword)
+
+    def _get_search_page_by_url(self, url: str) -> BeautifulSoup:
+        r = requests.get(url, headers=self._request_headers)
+
+        if r.encoding == 'ISO-8859-1':
+            encodings = requests.utils.get_encodings_from_content(r.text)
+            if encodings:
+                encoding = encodings[0]
+            else:
+                encoding = r.apparent_encoding
+            encode_content = r.content.decode(encoding, 'ignore')
+        else:
+            encode_content = r.text
+        soup = BeautifulSoup(encode_content, features="lxml")
+        return soup
+
+    def _get_next_page_url(self, soup: BeautifulSoup) -> Optional[str]:
+        selector = "div.pagebox a[title='下一页']"
+        next_page_tag = soup.select_one(selector)
+        if next_page_tag is None:
+            return None
+        next_page_link_text = next_page_tag["href"]
+        pattern = re.compile("javascript:linkPost\\('(.*)','(.*)'\\)")
+        result = re.search(pattern, next_page_link_text)
+        if result is None:
+            return None
+        return "https://search.sina.com.cn" + result.group(1) + result.group(2)

searchpageextractor/SogouSearchPageExtractor.py

@@ -0,0 +1,118 @@
+import concurrent.futures
+import re
+from typing import List, Optional
+
+import requests
+from bs4 import BeautifulSoup
+
+from model.news import News
+from pageextractor.util import get_extractor
+from searchpageextractor.SearchPageExtractor import SearchPageExtractor
+
+
+class SogouSearchPageExtractor(SearchPageExtractor):
+    _search_base_url = "https://www.sogou.com/sogou?interation=1728053249&query={}"
+    _next_page_base_url = "https://www.sogou.com/sogou"
+    _base_url = "https://www.sogou.com"
+    _request_headers = {"User-Agent":
+                            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
+                            "Chrome/94.0.4606.61 Safari/537.36"}
+
+    def search_by_keyword(self, keyword: str, max_page: int) -> List[News]:
+        all_item_links = self._get_all_item_links(keyword, max_page)
+        search_result = []
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = []
+            for link in all_item_links:
+                page_extractor = get_extractor(link)
+                if page_extractor is None:
+                    continue
+                futures.append(executor.submit(page_extractor.get_news))
+            for future in concurrent.futures.as_completed(futures):
+                r = future.result()
+                if r is not None:
+                    search_result.append(r)
+        return search_result
+
+    def _get_all_item_links(self, keyword: str, max_page: int) -> List[str]:
+        url_list = []
+        current_page = 1
+        search_page = self._get_search_page(keyword)
+        item_urls = self._get_item_urls(search_page)
+        url_list.extend(item_urls)
+        next_page_url = self._get_next_page_url(search_page)
+
+        while next_page_url is not None and current_page < max_page:
+            search_page = self._get_search_page_by_url(next_page_url)
+            item_urls = self._get_item_urls(search_page)
+            url_list.extend(item_urls)
+            next_page_url = self._get_next_page_url(search_page)
+            current_page += 1
+        return url_list
+
+    def _get_search_url(self, keyword: str) -> str:
+        return self._search_base_url.format(keyword)
+
+    def _get_search_page(self, keyword: str) -> BeautifulSoup:
+        search_url = self._get_search_url(keyword)
+        return self._get_search_page_by_url(search_url)
+
+    def _get_search_page_by_url(self, url: str) -> BeautifulSoup:
+        r = requests.get(url, headers=self._request_headers)
+
+        if r.encoding == 'ISO-8859-1':
+            encodings = requests.utils.get_encodings_from_content(r.text)
+            if encodings:
+                encoding = encodings[0]
+            else:
+                encoding = r.apparent_encoding
+            encode_content = r.content.decode(encoding, 'ignore')
+        else:
+            encode_content = r.text
+        soup = BeautifulSoup(encode_content, features="lxml")
+        return soup
+
+    def _get_next_page_url(self, soup: BeautifulSoup) -> Optional[str]:
+        selector = "a#sogou_next"
+        next_page_tag = soup.select_one(selector)
+        if next_page_tag is None:
+            return None
+        return self._next_page_base_url + next_page_tag["href"]
+
+    def _get_item_urls(self, soup: BeautifulSoup) -> List[str]:
+        selector = "div.vrwrap h3.vr-title>a"
+        item_links = soup.select(selector)
+        result_urls = []
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [executor.submit(convert_url, self._base_url + link["href"]) for link in item_links]
+            for future in concurrent.futures.as_completed(futures):
+                result = future.result()
+                if result is not None:
+                    result_urls.append(result)
+        return result_urls
+
+
+def convert_url(source_url: str):
+    r = requests.get(source_url)
+    if r.encoding == 'ISO-8859-1':
+        encodings = requests.utils.get_encodings_from_content(r.text)
+        if encodings:
+            encoding = encodings[0]
+        else:
+            encoding = r.apparent_encoding
+        encode_content = r.content.decode(encoding, 'ignore')
+    else:
+        encode_content = r.text
+    soup = BeautifulSoup(encode_content, features="lxml")
+    script_str = soup.select_one("script").decode_contents()
+    return extract_redirect_url(script_str)
+
+
+def extract_redirect_url(script: str):
+    pattern = re.compile("window\\.location\\.replace\\(\"(.*)\"\\)")
+    result = re.search(pattern, script)
+    if result is None:
+        return None
+    return result.group(1)