-
Notifications
You must be signed in to change notification settings - Fork 10
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
build: add setuptools for python3.12+ as a dependency
- Loading branch information
Showing
10 changed files
with
163 additions
and
174 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -22,7 +22,7 @@ | |
**/__pycache__ | ||
|
||
# venv | ||
venv | ||
venv* | ||
|
||
# local do remote package test | ||
local_use | ||
|
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
27 changes: 27 additions & 0 deletions
27
playground/parse_js/dig-js-fetching/response_404_is_false.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
import requests | ||
|
||
url = "https://tw.linovelib.com/themes/zhmb/js/hm.js" | ||
|
||
headers = { | ||
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", | ||
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6", | ||
"cache-control": "max-age=0", | ||
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0" | ||
} | ||
# len() | ||
|
||
response = requests.get(url, headers=headers, timeout=20) | ||
|
||
# response.__bool__() -> response.ok | ||
|
||
# response.ok | ||
# Returns True if :attr:`status_code` is less than 400, False if not. | ||
|
||
# 404 -> False | ||
|
||
if response: | ||
print('1') | ||
else: | ||
print('2') | ||
|
||
# 2 |
52 changes: 52 additions & 0 deletions
52
playground/parse_js/dig-js-fetching/use_aiohttp_multi_reqs.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
import asyncio | ||
from typing import Tuple | ||
|
||
import requests | ||
|
||
from linovelib2epub.logger import Logger | ||
from linovelib2epub.utils import requests_get_with_retry | ||
|
||
logger = Logger(logger_name=__name__, | ||
logger_level='DEBUG').get_logger() | ||
|
||
url1 = "https://tw.linovelib.com/themes/zhmb/js/hm.js" | ||
url2 = "https://tw.linovelib.com/themes/zhmb/js/readtool.js" | ||
urls = [url1, url2] | ||
|
||
headers = { | ||
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", | ||
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6", | ||
"cache-control": "max-age=0", | ||
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0" | ||
} | ||
|
||
|
||
def fetch_with_retry(url) -> Tuple[str, str | None]: | ||
resp = requests_get_with_retry(requests, url, headers=headers, logger=logger) | ||
if resp: | ||
return url, resp.text | ||
|
||
return url, None | ||
|
||
|
||
async def fetch_async(url, fetch_func): | ||
loop = asyncio.get_event_loop() | ||
return await loop.run_in_executor(None, fetch_func, url) | ||
|
||
|
||
async def fetch_js(urls): | ||
tasks = [asyncio.create_task(fetch_async(url, fetch_with_retry)) for url in urls] | ||
completed, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) | ||
|
||
for task in completed: | ||
url, resp_text = task.result() | ||
if resp_text: | ||
return url, resp_text | ||
|
||
# 没有任何一个url能返回文本 | ||
return None, None | ||
|
||
|
||
if __name__ == '__main__': | ||
url, text = asyncio.run(fetch_js(urls)) | ||
print(url, text) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
import requests | ||
|
||
|
||
headers = { | ||
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", | ||
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6", | ||
"cache-control": "max-age=0", | ||
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0" | ||
} | ||
url1 = "https://tw.linovelib.com/themes/zhmb/js/hm.js" | ||
# url = "https://tw.linovelib.com/themes/zhmb/js/readtool.js" | ||
response = requests.get(url1, headers=headers) | ||
|
||
print(response.text) | ||
print(response.status_code) |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.