Compare commits

...

5 Commits

3 changed files with 13 additions and 6 deletions

2
HBD.py
View File

@@ -1,5 +1,5 @@
from pymongo import MongoClient
#wdawfqfqwfqwfqw
def connect_to_mongo():
"""Подключение к MongoDB"""

View File

@@ -2,12 +2,13 @@ from bs4 import BeautifulSoup as bs
import requests
import HBD
link = 'https://x3.h-chan.me/manga/newest?offset='
link = 'https://x8.h-chan.me/manga/newest?offset=' #https://x8.h-chan.me/manga/
result = HBD.find_doc_with_max_id(HBD.connect_to_mongo())
max_id = '51196'
max_num = 29915
def form_date(date_str):
months = {
"января": "01",
@@ -47,7 +48,7 @@ def try_request(link, max_retries=50):
except:
retries += 1
def pars(link, flag):
def pars(link, flag, count):
data_hantai = {}
response = try_request(link)
@@ -93,16 +94,22 @@ def pars(link, flag):
'original_id' : ID,
}
print(f'стр - {count}', data_hantai)
for i, j in data_hantai.items():
print('-'*10, i, j['manga_link'])
print(data_hantai)
return data_hantai, flag
def get_data():
data = {}
count = 1
for i in range(0, 5000, 20):
flag = False
data_hantai, flag = pars(link + str(i), flag)
data_hantai, flag = pars(link + str(i), flag, count)
count +=1
data.update(data_hantai)

View File

@@ -2,7 +2,7 @@ from bs4 import BeautifulSoup as bs
import requests
import re
#Поиск фулл страниц манги
#Поиск фулл страниц манги цауц
def process_hentai_data(hent_data, max_retries=50):
data_hantai = {}