Skip to content

Commit

Permalink
build: add setuptools for python3.12+ as a dependency
Browse files Browse the repository at this point in the history
  • Loading branch information
wdpm committed Oct 30, 2024
1 parent 7dd83a5 commit ad3b456
Show file tree
Hide file tree
Showing 10 changed files with 163 additions and 174 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
**/__pycache__

# venv
venv
venv*

# local do remote package test
local_use
Expand Down
41 changes: 0 additions & 41 deletions playground/optimize-mapping-rule-fetching/fetch_js.py

This file was deleted.

31 changes: 0 additions & 31 deletions playground/optimize-mapping-rule-fetching/index.py

This file was deleted.

27 changes: 27 additions & 0 deletions playground/parse_js/dig-js-fetching/response_404_is_false.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import requests

url = "https://tw.linovelib.com/themes/zhmb/js/hm.js"

headers = {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
"cache-control": "max-age=0",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0"
}
# len()

response = requests.get(url, headers=headers, timeout=20)

# response.__bool__() -> response.ok

# response.ok
# Returns True if :attr:`status_code` is less than 400, False if not.

# 404 -> False

if response:
print('1')
else:
print('2')

# 2
52 changes: 52 additions & 0 deletions playground/parse_js/dig-js-fetching/use_aiohttp_multi_reqs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
import asyncio
from typing import Tuple

import requests

from linovelib2epub.logger import Logger
from linovelib2epub.utils import requests_get_with_retry

logger = Logger(logger_name=__name__,
logger_level='DEBUG').get_logger()

url1 = "https://tw.linovelib.com/themes/zhmb/js/hm.js"
url2 = "https://tw.linovelib.com/themes/zhmb/js/readtool.js"
urls = [url1, url2]

headers = {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
"cache-control": "max-age=0",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0"
}


def fetch_with_retry(url) -> Tuple[str, str | None]:
resp = requests_get_with_retry(requests, url, headers=headers, logger=logger)
if resp:
return url, resp.text

return url, None


async def fetch_async(url, fetch_func):
loop = asyncio.get_event_loop()
return await loop.run_in_executor(None, fetch_func, url)


async def fetch_js(urls):
tasks = [asyncio.create_task(fetch_async(url, fetch_with_retry)) for url in urls]
completed, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)

for task in completed:
url, resp_text = task.result()
if resp_text:
return url, resp_text

# 没有任何一个url能返回文本
return None, None


if __name__ == '__main__':
url, text = asyncio.run(fetch_js(urls))
print(url, text)
15 changes: 15 additions & 0 deletions playground/parse_js/dig-js-fetching/use_requests.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import requests


headers = {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
"cache-control": "max-age=0",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0"
}
url1 = "https://tw.linovelib.com/themes/zhmb/js/hm.js"
# url = "https://tw.linovelib.com/themes/zhmb/js/readtool.js"
response = requests.get(url1, headers=headers)

print(response.text)
print(response.status_code)
49 changes: 0 additions & 49 deletions playground/parse_js/probe_js_file.py

This file was deleted.

5 changes: 4 additions & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# after python3.12
setuptools

# project dependencies
aiofiles==23.1.0
aiohttp==3.10.2
brotli==1.1.0
Expand All @@ -8,7 +12,6 @@ dynaconf==3.2.3
EbookLib==0.17.1
esprima==4.0.1
fake-useragent==1.1.1
# importlib-resources==6.4.5
inquirer==3.1.2
lxml==5.3.0
pillow==11.0.0
Expand Down
Loading

0 comments on commit ad3b456

Please sign in to comment.