110 lines
2.5 KiB
Python
110 lines
2.5 KiB
Python
import requests
|
|
from bs4 import BeautifulSoup as bs
|
|
import function as f
|
|
import json
|
|
import os
|
|
import log_err as lg
|
|
|
|
import import_in_BD as ib
|
|
|
|
link = 'https://povar.ru/list/'
|
|
buffer = []
|
|
|
|
def buffer_import():
|
|
global buffer
|
|
|
|
try:
|
|
ib.bulk_write_recipes(buffer)
|
|
buffer.clear()
|
|
except Exception as bulk_e:
|
|
print(f"⚠️ Bulk-ошибка: {bulk_e}. Переключаемся на поштучную запись...")
|
|
for recipe in buffer:
|
|
try:
|
|
ib.import_json_in_mongo(recipe) # ← ОДИН рецепт
|
|
except Exception as single_e:
|
|
lg.log_error(recipe.get('url', 'unknown'), f"MongoDB-фейл: {single_e}")
|
|
buffer.clear()
|
|
|
|
|
|
|
|
|
|
|
|
def open_url():
|
|
total_type_recip = json.load(open('unique_urls.json', 'r', encoding='utf-8'))
|
|
|
|
for url in total_type_recip:
|
|
if len(buffer) >= 200:
|
|
print('❗️ Сейвим', len(buffer))
|
|
print(buffer)
|
|
buffer_import()
|
|
|
|
pars_recipie(url)
|
|
|
|
buffer_import()
|
|
|
|
|
|
def pars_recipie(url='https://povar.ru/recipes/slivochnaya_karbonara-73186.html'):
|
|
|
|
try:
|
|
|
|
response = f.try_request(url)
|
|
soup = bs(response.text, 'html.parser')
|
|
|
|
recipe_name = soup.find(class_='detailed fn').text
|
|
|
|
main_container = soup.find(class_='cont_area hrecipe')
|
|
|
|
|
|
steps = f.extr_steps(main_container) #Сборка шагов
|
|
|
|
if steps is None:
|
|
lg.log_error(url, 'Нет шагов')
|
|
return None
|
|
|
|
name_id = url.split('/')[-1].strip()
|
|
try:
|
|
name_id = name_id.replace('.html', '').strip()
|
|
except: pass
|
|
|
|
photo = main_container.find(class_='photo').get('src')
|
|
|
|
|
|
detailed_tags = f.extract_tags_from_detailed_tags(main_container) #Собираем теги
|
|
ingredients = f.extr_ingredient(main_container) #Собираем ингредиенты
|
|
calories_info = f.extract_nutrition(main_container.find_all(class_='circle')) #БЖУ
|
|
|
|
|
|
recip = {'_id' : name_id,
|
|
'recipe_name':recipe_name,
|
|
'url':url,
|
|
'preview_img':photo,
|
|
'tags':detailed_tags,
|
|
'ingredients':ingredients,
|
|
'nutritional_value':calories_info,
|
|
'steps':steps}
|
|
|
|
print('⭕',recipe_name)
|
|
print('🤍',recip)
|
|
|
|
buffer.append(recip)
|
|
|
|
|
|
except Exception as e:
|
|
print(url, e)
|
|
lg.log_error(url, e)
|
|
|
|
|
|
#pars_recipie()
|
|
open_url()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|