diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..8b1a44e8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,301 @@ +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# Python +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +# PyPI configuration file +.pypirc + +# Visual Studio Code +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +# Virtualenv +# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ +.Python +[Bb]in +[Ii]nclude +[Ll]ib +[Ll]ib64 +[Ll]ocal +[Ss]cripts +pyvenv.cfg +.venv +pip-selfcheck.json + +# CodeGPT plugin for VSCode +.codegpt + +# scraping output files +output/ diff --git a/.streamlit/config.toml b/.streamlit/config.toml new file mode 100644 index 00000000..658868ff --- /dev/null +++ b/.streamlit/config.toml @@ -0,0 +1,2 @@ +[browser] +gatherUsageStats = false diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..b0948629 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,24 @@ +{ + // Use IntelliSense para saber los atributos posibles. + // Mantenga el puntero para ver las descripciones de los existentes atributos. + // Para más información, visite: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Streamlit APP", + "type": "debugpy", + "request": "launch", + "module": "streamlit", + "args": ["run", "streamlit_app.py"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Python: This file", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal" + } + ] +} diff --git a/README.md b/README.md index 12c2821c..63480f59 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ ScrapeMaster is a Streamlit-based web scraping application designed to simplify ## Prerequisites Before you begin, ensure you have the following installed: + - Python 3.6 or higher - Pip for managing Python packages @@ -38,6 +39,31 @@ source venv/bin/activate pip install -r requirements.txt ``` +### Streamlit opt-out configuration (optative) + +Setting global config file: + +- Linux/macOS: + + ```bash + # create directory if it doesn't exists + mkdir -p ~/.streamlit/ + # Setting global config file + cp ./.streamlit/config.toml ~/.streamlit/config.toml + ``` + +- Windows: + + ```powershell + # create directory if it doesn't exists + $targetDir = "$env:USERPROFILE\.streamlit\" + if (-not (Test-Path -Path $targetDir)) { + New-Item -ItemType Directory -Path $targetDir + } + # Setting global config file + Copy-Item -Path ".\.streamlit\config.toml" -Destination $targetDir + ``` + ## Launching the Application To run ScrapeMaster, navigate to the project directory and run the following command: @@ -46,6 +72,11 @@ To run ScrapeMaster, navigate to the project directory and run the following com streamlit run streamlit_app.py ``` - ## Usage -After launching the application, open your web browser to the indicated address (typically http://localhost:8501). Use the sidebar to input the URL and fields you wish to scrape, then click the "Scrape" button to see results. \ No newline at end of file + +After launching the application, open your web browser to the indicated address (typically ). Use the sidebar to input the URL and fields you wish to scrape, then click the "Scrape" button to see results. + +## References + +- +- diff --git a/api_management.py b/api_management.py index e66a3cfc..7fc8a140 100644 --- a/api_management.py +++ b/api_management.py @@ -1,8 +1,11 @@ -import streamlit as st +#!/usr/bin/env python3 + import os +import streamlit as st + def get_api_key(api_key_name): - # Check if the API key from the sidebar is present, else fallback to the .env file + """Check if the API key from the sidebar is present, else fallback to the .env file""" if api_key_name == 'OPENAI_API_KEY': return st.session_state['openai_api_key'] or os.getenv(api_key_name) elif api_key_name == 'GOOGLE_API_KEY': diff --git a/assets.py b/assets.py index 3c8cebf1..4f0f507c 100644 --- a/assets.py +++ b/assets.py @@ -1,10 +1,12 @@ +#!/usr/bin/env python3 + """ This module contains configuration variables and constants that are used across different parts of the application. """ # List of user agents to mimic different users -USER_AGENTS = [ +USER_AGENTS = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0", @@ -65,24 +67,24 @@ # Define the pricing for models without Batch API PRICING = { "gpt-4o-mini": { - "input": 0.150 / 1_000_000, # $0.150 per 1M input tokens - "output": 0.600 / 1_000_000, # $0.600 per 1M output tokens + "input": 0.150 / 1_000_000, # $0.150 per 1M input tokens + "output": 0.600 / 1_000_000, # $0.600 per 1M output tokens }, "gpt-4o-2024-08-06": { "input": 2.5 / 1_000_000, # $2.5 per 1M input tokens - "output": 10 / 1_000_000, # $10 per 1M output tokens + "output": 10 / 1_000_000, # $10 per 1M output tokens }, "gemini-1.5-flash": { "input": 0.075 / 1_000_000, # $0.075 per 1M input tokens - "output": 0.30 / 1_000_000, # $0.30 per 1M output tokens + "output": 0.30 / 1_000_000, # $0.30 per 1M output tokens }, "Llama3.1 8B": { - "input": 0 , # Free - "output": 0 , # Free + "input": 0, # Free + "output": 0, # Free }, "Groq Llama3.1 70b": { - "input": 0 , # Free - "output": 0 , # Free + "input": 0, # Free + "output": 0, # Free }, # Add other models and their prices here if needed } @@ -94,47 +96,59 @@ } # Other reusable constants or configuration settings -HEADLESS_OPTIONS = ["--disable-gpu", "--disable-dev-shm-usage","--window-size=1920,1080","--disable-search-engine-choice-screen","--disable-blink-features=AutomationControlled"] - - -HEADLESS_OPTIONS_DOCKER = ["--headless=new","--no-sandbox","--disable-gpu", "--disable-dev-shm-usage","--disable-software-rasterizer","--disable-setuid-sandbox","--remote-debugging-port=9222","--disable-search-engine-choice-screen"] -#in case you don't need to open the website -##HEADLESS_OPTIONS=HEADLESS_OPTIONS+[ "--headless=new"] - -#number of scrolls -NUMBER_SCROLL=2 +HEADLESS_OPTIONS = [ + "--disable-blink-features=AutomationControlled", + "--disable-dev-shm-usage", + "--disable-gpu", + "--disable-search-engine-choice-screen", + "--window-size=1920,1080" +] +HEADLESS_OPTIONS_DOCKER = [ + "--disable-dev-shm-usage", + "--disable-gpu", + "--disable-search-engine-choice-screen", + "--disable-setuid-sandbox", + "--disable-software-rasterizer", + "--headless=new", + "--no-sandbox", + "--remote-debugging-port=9222" +] -LLAMA_MODEL_FULLNAME="lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF" -GROQ_LLAMA_MODEL_FULLNAME="llama-3.1-70b-versatile" +# in case you don't need to open the website +# HEADLESS_OPTIONS=HEADLESS_OPTIONS+[ "--headless=new"] -SYSTEM_MESSAGE = """You are an intelligent text extraction and conversion assistant. Your task is to extract structured information - from the given text and convert it into a pure JSON format. The JSON should contain only the structured data extracted from the text, - with no additional commentary, explanations, or extraneous information. - You could encounter cases where you can't find the data of the fields you have to extract or the data will be in a foreign language. - Please process the following text and provide the output in pure JSON format with no words before or after the JSON:""" +# number of scrolls +NUMBER_SCROLL = 2 -USER_MESSAGE = f"Extract the following information from the provided text:\nPage content:\n\n" - +LLAMA_MODEL_FULLNAME = "lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF" +GROQ_LLAMA_MODEL_FULLNAME = "llama-3.1-70b-versatile" +SYSTEM_MESSAGE = """ +You are an intelligent text extraction and conversion assistant. Your task is to extract structured information +from the given text and convert it into a pure JSON format. The JSON should contain only the structured data extracted from the text, +with no additional commentary, explanations, or extraneous information. +You could encounter cases where you can't find the data of the fields you have to extract or the data will be in a foreign language. +Please process the following text and provide the output in pure JSON format with no words before or after the JSON: +""" +USER_MESSAGE = "Extract the following information from the provided text:\nPage content:\n\n" PROMPT_PAGINATION = """ -You are an assistant that extracts pagination elements from markdown content of websites your goal as a universal pagination scrapper of urls from all websites no matter how different they are. +You are an assistant that extracts pagination elements from markdown content of websites. Your goal as a universal pagination scrapper of urls from all websites, no matter how different they are. -Please extract the following: +Extract the following: -- The url of the 'Next', 'More', 'See more', 'load more' or any other button indicating how to access the next page, if any, it should be 1 url and no more, if there are multiple urls with the same structure leave this empty. +- The url of the 'Next', 'More', 'See more', 'load more' or any other button indicating how to access the next page, if any: It should be one url and no more, if there are multiple urls with the same structure, discard this url from output. -- A list of page URLs for pagination it should be a pattern of similar urls with pages that are numbered, if you detect this pattern and the numbers starts from a certain low number until a large number generate the rest of the urls even if they're not included, -your goal here is to give as many urls for the user to choose from in order for them to do further scraping, you will have to deal with very different websites that can potientially have so many urls of images and other elements, -detect only the urls that are clearly defining a pattern to show data on multiple pages, sometimes there is only a part of these urls and you have to combine it with the initial url, that will be provided for you at the end of this prompt. +- A list of page URLs for pagination: It should be a pattern of similar urls with pages that are numbered, if you detect this pattern and the numbers starts from a certain low number until a large number generate the rest of the urls even if they're not included. + Your goal here is to give as many urls for the user to choose from in order for them to do further scraping, you will have to deal with very different websites that can potentially have so many urls of images and other elements. + Detect only the urls that are clearly defining a pattern to show data on multiple pages. Sometimes there is only a part of these urls and you have to combine it with the initial url, that will be provided for you at the end of this prompt. -- The user can give you indications on how the pagination works for the specific website at the end of this prompt, if those indications are not empty pay special attention to them as they will directly help you understand the structure and the number of pages to generate. +The user can give you indications on how the pagination works for the specific website at the end of this prompt. If those indications are not empty, pay special attention to them, as they will directly help you understand the structure and the number of pages to generate. Provide the output as a JSON object with the following structure: - { "page_urls": ["url1", "url2", "url3",...,"urlN"] } diff --git a/changes.md b/changes.md new file mode 100644 index 00000000..330fee9f --- /dev/null +++ b/changes.md @@ -0,0 +1,42 @@ +# Changes Log + +## Latest changes + +Updated change log + +## Commit 1fd6cd9 - Jan 20, 2025 + +Add change log & app debug config + +- Added `changes.md` file: to maintain changes log. +- Added streamlit app debug configuration for VSCode. + +## Commit d4e9ec9 - Jan 15, 2025 + +- Bugfix: migrated to v1.0 call + When using generic openai-compatible model, the chat completation call is using `client` object as recommended in v1.0 API. + +## Commit 6fef093 - Jan 14, 2025 + +Prompt improvement & some linting. + +- Modifications on scaper and pagination Prompts, to be clearer for the LLM. +- Added links to Youtube videos for this project in `README.md`, as [`silasneo/scrape-master`](https://github.com/silasneo/scrape-master) fork did. + +## Commit 6fb318d - Jan 14, 2025 + +- Bugfix: duplicated text in pagination prompt. + Removed error of adding twice the main pagination prompt text when scraping for pagination links with a LLM call. + +## Commit de25815 - Jan 13, 2025 + +- Formatting & linting on all files. + Includes removal of unused imports, removal of old comments, added header with she-bang for python, formatted python code for better readability, corrected problems with f-strings and logging options, etc. + +## Commit 9a88bbb - Jan 9, 2025 + +- Created `.gitignore` file with exclusions for `output` dir, and common cases for python, virtual environment, windows, vscode and pycharm. + Hint from [`amgrbi96/AI-scraper`](https://github.com/amgrbi96/AI-scraper) fork. +- Streamlit opt-out configuration added. +- Ordered contents of `requirements.txt` files. +- Removed empty `test.py` file. diff --git a/pagination_detector.py b/pagination_detector.py index ac3ada03..9454f822 100644 --- a/pagination_detector.py +++ b/pagination_detector.py @@ -1,31 +1,41 @@ -# pagination_detector.py +#!/usr/bin/env python3 -import os +import logging import json from typing import List, Dict, Tuple, Union -from pydantic import BaseModel, Field, ValidationError - -import tiktoken from dotenv import load_dotenv +from pydantic import BaseModel, Field + from openai import OpenAI +import tiktoken + import google.generativeai as genai from groq import Groq - from api_management import get_api_key -from assets import PROMPT_PAGINATION, PRICING, LLAMA_MODEL_FULLNAME, GROQ_LLAMA_MODEL_FULLNAME - +from assets import ( + GROQ_LLAMA_MODEL_FULLNAME, + LLAMA_MODEL_FULLNAME, + PRICING, + PROMPT_PAGINATION +) + +# Load environment variables, expecting API Keys load_dotenv() -import logging + class PaginationData(BaseModel): - page_urls: List[str] = Field(default_factory=list, description="List of pagination URLs, including 'Next' button URL if present") + page_urls: List[str] = Field( + default_factory=list, + description="List of pagination URLs, including 'Next' button URL if present" + ) + def calculate_pagination_price(token_counts: Dict[str, int], model: str) -> float: """ Calculate the price for pagination based on token counts and the selected model. - + Args: token_counts (Dict[str, int]): A dictionary containing 'input_tokens' and 'output_tokens'. model (str): The name of the selected model. @@ -35,29 +45,34 @@ def calculate_pagination_price(token_counts: Dict[str, int], model: str) -> floa """ input_tokens = token_counts['input_tokens'] output_tokens = token_counts['output_tokens'] - + input_price = input_tokens * PRICING[model]['input'] output_price = output_tokens * PRICING[model]['output'] - + return input_price + output_price + def detect_pagination_elements(url: str, indications: str, selected_model: str, markdown_content: str) -> Tuple[Union[PaginationData, Dict, str], Dict, float]: - try: - """ - Uses AI models to analyze markdown content and extract pagination elements. + """ + Uses AI models to analyze markdown content and extract pagination elements. - Args: - selected_model (str): The name of the OpenAI model to use. - markdown_content (str): The markdown content to analyze. + Args: + selected_model (str): The name of the OpenAI model to use. + markdown_content (str): The markdown content to analyze. - Returns: - Tuple[PaginationData, Dict, float]: Parsed pagination data, token counts, and pagination price. - """ - prompt_pagination = PROMPT_PAGINATION+"\n The url of the page to extract pagination from "+url+"if the urls that you find are not complete combine them intelligently in a way that fit the pattern **ALWAYS GIVE A FULL URL**" + Returns: + Tuple[PaginationData, Dict, float]: + Parsed pagination data, token counts, and pagination price. + """ + try: + prompt_pagination = PROMPT_PAGINATION + \ + "\nThe url of the page to extract pagination from: '" + url + \ + "'\nIf the urls that you find are not complete, combine them intelligently in a way that fit the pattern. **ALWAYS GIVE A FULL URL**" if indications != "": - prompt_pagination +=PROMPT_PAGINATION+"\n\n these are the users indications that, pay special attention to them: "+indications+"\n\n below are the markdowns of the website: \n\n" + prompt_pagination += "\n\nThese are the users indications that, pay special attention to them: " + \ + indications + "\n\nBelow are the markdowns of the website:\n\n" else: - prompt_pagination +=PROMPT_PAGINATION+"\n There are no user indications in this case just apply the logic described. \n\n below are the markdowns of the website: \n\n" + prompt_pagination += "\nThere are no user indications in this case just apply the logic described.\n\nBelow are the markdowns of the website:\n\n" if selected_model in ["gpt-4o-mini", "gpt-4o-2024-08-06"]: # Use OpenAI API @@ -77,14 +92,18 @@ def detect_pagination_elements(url: str, indications: str, selected_model: str, # Calculate tokens using tiktoken encoder = tiktoken.encoding_for_model(selected_model) input_token_count = len(encoder.encode(markdown_content)) - output_token_count = len(encoder.encode(json.dumps(parsed_response.dict()))) + output_token_count = len(encoder.encode( + json.dumps(parsed_response.dict()))) token_counts = { "input_tokens": input_token_count, "output_tokens": output_token_count } # Calculate the price - pagination_price = calculate_pagination_price(token_counts, selected_model) + pagination_price = calculate_pagination_price( + token_counts, + selected_model + ) return parsed_response, token_counts, pagination_price @@ -110,11 +129,13 @@ def detect_pagination_elements(url: str, indications: str, selected_model: str, } # Get the result response_content = completion.text - + # Log the response content and its type - logging.info(f"Gemini Flash response type: {type(response_content)}") - logging.info(f"Gemini Flash response content: {response_content}") - + logging.info("Gemini Flash response type: %s", + type(response_content)) + logging.info("Gemini Flash response content: %s", + response_content) + # Try to parse the response as JSON try: parsed_data = json.loads(response_content) @@ -127,35 +148,46 @@ def detect_pagination_elements(url: str, indications: str, selected_model: str, pagination_data = PaginationData(page_urls=[]) # Calculate the price - pagination_price = calculate_pagination_price(token_counts, selected_model) + pagination_price = calculate_pagination_price( + token_counts, + selected_model + ) return pagination_data, token_counts, pagination_price elif selected_model == "Llama3.1 8B": # Use Llama model via OpenAI API pointing to local server - openai.api_key = "lm-studio" - openai.api_base = "http://localhost:1234/v1" - response = openai.ChatCompletion.create( + client = OpenAI( + api_key="lm-studio", + base_url="http://localhost:1234/v1" + ) + + response = client.chat.completions.create( model=LLAMA_MODEL_FULLNAME, messages=[ {"role": "system", "content": prompt_pagination}, {"role": "user", "content": markdown_content}, ], - temperature=0.7, + temperature=0.7 ) response_content = response['choices'][0]['message']['content'].strip() - # Try to parse the JSON try: pagination_data = json.loads(response_content) except json.JSONDecodeError: - pagination_data = {"next_buttons": [], "page_urls": []} + pagination_data = { + "next_buttons": [], + "page_urls": [] + } # Token counts token_counts = { "input_tokens": response['usage']['prompt_tokens'], "output_tokens": response['usage']['completion_tokens'] } # Calculate the price - pagination_price = calculate_pagination_price(token_counts, selected_model) + pagination_price = calculate_pagination_price( + token_counts, + selected_model + ) return pagination_data, token_counts, pagination_price @@ -181,7 +213,10 @@ def detect_pagination_elements(url: str, indications: str, selected_model: str, "output_tokens": response.usage.completion_tokens } # Calculate the price - pagination_price = calculate_pagination_price(token_counts, selected_model) + pagination_price = calculate_pagination_price( + token_counts, + selected_model + ) '''# Ensure the pagination_data is a dictionary if isinstance(pagination_data, PaginationData): @@ -195,27 +230,27 @@ def detect_pagination_elements(url: str, indications: str, selected_model: str, raise ValueError(f"Unsupported model: {selected_model}") except Exception as e: - logging.error(f"An error occurred in detect_pagination_elements: {e}") + logging.error("An error occurred in detect_pagination_elements: %s", e) # Return default values if an error occurs return PaginationData(page_urls=[]), {"input_tokens": 0, "output_tokens": 0}, 0.0 if __name__ == "__main__": - url="""https://scrapeme.live/shop/""" + url = """https://scrapeme.live/shop/""" # Define the path to your markdown file markdown_file_path = r"C:\Users\redam\Documents\VSCode\ScrapeMaster2.0\output\scrapeme_live_2024_09_24__00_33_20\rawData_1.md" - + # Read the markdown content from the file with open(markdown_file_path, 'r', encoding='utf-8') as f: markdown_content = f.read() - + # Specify the model you want to use selected_model = 'gemini-1.5-flash' # Replace with your desired model # Call the detect_pagination_elements function - pagination_data, token_counts, pagination_price = detect_pagination_elements(url,"",selected_model, markdown_content) - + pagination_data, token_counts, pagination_price = detect_pagination_elements( + url, "", selected_model, markdown_content) + print("Page URLs:", pagination_data.page_urls) print("Pagination Price:", pagination_price) - diff --git a/requirements.txt b/requirements.txt index 8c2ec333..4330f39c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,16 @@ +beautifulsoup4 +google-generativeai +groq +html2text openai -python-dotenv +openpyxl pandas pydantic +python-dotenv +readability-lxml requests -beautifulsoup4 -html2text -tiktoken selenium -readability-lxml streamlit streamlit-tags -openpyxl -groq -google-generativeai +tiktoken webdriver-manager diff --git a/scraper.py b/scraper.py index b1aca0b2..1c8514a3 100644 --- a/scraper.py +++ b/scraper.py @@ -1,41 +1,44 @@ +#!/usr/bin/env python3 + import os import random import time import re import json from datetime import datetime -from typing import List, Dict, Type +from typing import List, Type +from dotenv import load_dotenv import pandas as pd from bs4 import BeautifulSoup -from pydantic import BaseModel, Field, create_model +from pydantic import BaseModel, create_model import html2text -import tiktoken -import streamlit as st -from dotenv import load_dotenv from selenium import webdriver from selenium.webdriver.chrome.service import Service from selenium.webdriver.chrome.options import Options -from selenium.webdriver.common.by import By -from selenium.webdriver.common.action_chains import ActionChains -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC from webdriver_manager.chrome import ChromeDriverManager - from openai import OpenAI +import tiktoken import google.generativeai as genai from groq import Groq from api_management import get_api_key -from assets import USER_AGENTS,PRICING,HEADLESS_OPTIONS,SYSTEM_MESSAGE,USER_MESSAGE,LLAMA_MODEL_FULLNAME,GROQ_LLAMA_MODEL_FULLNAME,HEADLESS_OPTIONS_DOCKER +from assets import ( + GROQ_LLAMA_MODEL_FULLNAME, + HEADLESS_OPTIONS, + HEADLESS_OPTIONS_DOCKER, + LLAMA_MODEL_FULLNAME, + PRICING, + SYSTEM_MESSAGE, + USER_MESSAGE +) + +# Load environment variables, expecting API Keys load_dotenv() -# Set up the Chrome WebDriver options - - def is_running_in_docker(): """ Detect if the app is running inside a Docker container. @@ -47,6 +50,7 @@ def is_running_in_docker(): except Exception: return False + def setup_selenium(attended_mode=False): options = Options() service = Service(ChromeDriverManager().install()) @@ -66,8 +70,6 @@ def setup_selenium(attended_mode=False): return driver - - def fetch_html_selenium(url, attended_mode=False, driver=None): if driver is None: driver = setup_selenium(attended_mode) @@ -83,11 +85,14 @@ def fetch_html_selenium(url, attended_mode=False, driver=None): try: if not attended_mode: # Add more realistic actions like scrolling - driver.execute_script("window.scrollTo(0, document.body.scrollHeight/2);") + driver.execute_script( + "window.scrollTo(0, document.body.scrollHeight/2);") time.sleep(random.uniform(1.1, 1.8)) - driver.execute_script("window.scrollTo(0, document.body.scrollHeight/1.2);") + driver.execute_script( + "window.scrollTo(0, document.body.scrollHeight/1.2);") time.sleep(random.uniform(1.1, 1.8)) - driver.execute_script("window.scrollTo(0, document.body.scrollHeight/1);") + driver.execute_script( + "window.scrollTo(0, document.body.scrollHeight/1);") time.sleep(random.uniform(1.1, 1.8)) # Get the page source from the current page html = driver.page_source @@ -97,11 +102,9 @@ def fetch_html_selenium(url, attended_mode=False, driver=None): driver.quit() - - def clean_html(html_content): soup = BeautifulSoup(html_content, 'html.parser') - + # Remove headers and footers based on common HTML tags or classes for element in soup.find_all(['header', 'footer']): element.decompose() # Remove these tags and their content @@ -111,18 +114,16 @@ def clean_html(html_content): def html_to_markdown_with_readability(html_content): - - cleaned_html = clean_html(html_content) - + cleaned_html = clean_html(html_content) + # Convert to markdown markdown_converter = html2text.HTML2Text() markdown_converter.ignore_links = False markdown_content = markdown_converter.handle(cleaned_html) - + return markdown_content - def save_raw_data(raw_data: str, output_folder: str, file_name: str): """Save raw markdown data to the specified output folder.""" os.makedirs(output_folder, exist_ok=True) @@ -148,9 +149,10 @@ def create_listings_container_model(listing_model: Type[BaseModel]) -> Type[Base """ Create a container model that holds a list of the given listing model. """ - return create_model('DynamicListingsContainer', listings=(List[listing_model], ...)) - - + return create_model( + 'DynamicListingsContainer', + listings=(List[listing_model], ...) + ) def trim_to_token_limit(text, model, max_tokens=120000): @@ -161,6 +163,7 @@ def trim_to_token_limit(text, model, max_tokens=120000): return trimmed_text return text + def generate_system_message(listing_model: BaseModel) -> str: """ Dynamically generate a system message based on the fields in the provided listing model. @@ -180,13 +183,12 @@ def generate_system_message(listing_model: BaseModel) -> str: # Generate the system message dynamically system_message = f""" - You are an intelligent text extraction and conversion assistant. Your task is to extract structured information - from the given text and convert it into a pure JSON format. The JSON should contain only the structured data extracted from the text, - with no additional commentary, explanations, or extraneous information. - You could encounter cases where you can't find the data of the fields you have to extract or the data will be in a foreign language. - Please process the following text and provide the output in pure JSON format with no words before or after the JSON: - Please ensure the output strictly follows this schema: - + You are an intelligent text extraction and conversion assistant. Your task is to extract structured information + from the given text and convert it into a pure JSON format. The JSON should contain only the structured data extracted from the text, + with no additional commentary, explanations, or extraneous information. + You could encounter cases where you can't find the data of the fields you have to extract or the data will be in a foreign language. + Please process the following text and provide the output in pure JSON format with no words before or after the JSON. + Ensure the output strictly follows this schema: {{ "listings": [ {{ @@ -198,10 +200,9 @@ def generate_system_message(listing_model: BaseModel) -> str: return system_message - -def format_data(data, DynamicListingsContainer, DynamicListingModel, selected_model): +def format_data(data, dynamic_listings_container, dynamic_listing_model, selected_model): token_counts = {} - + if selected_model in ["gpt-4o-mini", "gpt-4o-2024-08-06"]: # Use OpenAI API client = OpenAI(api_key=get_api_key('OPENAI_API_KEY')) @@ -211,12 +212,13 @@ def format_data(data, DynamicListingsContainer, DynamicListingModel, selected_mo {"role": "system", "content": SYSTEM_MESSAGE}, {"role": "user", "content": USER_MESSAGE + data}, ], - response_format=DynamicListingsContainer + response_format=dynamic_listings_container ) # Calculate tokens using tiktoken encoder = tiktoken.encoding_for_model(selected_model) input_token_count = len(encoder.encode(USER_MESSAGE + data)) - output_token_count = len(encoder.encode(json.dumps(completion.choices[0].message.parsed.dict()))) + output_token_count = len(encoder.encode( + json.dumps(completion.choices[0].message.parsed.dict()))) token_counts = { "input_tokens": input_token_count, "output_tokens": output_token_count @@ -226,11 +228,13 @@ def format_data(data, DynamicListingsContainer, DynamicListingModel, selected_mo elif selected_model == "gemini-1.5-flash": # Use Google Gemini API genai.configure(api_key=get_api_key("GOOGLE_API_KEY")) - model = genai.GenerativeModel('gemini-1.5-flash', - generation_config={ - "response_mime_type": "application/json", - "response_schema": DynamicListingsContainer - }) + model = genai.GenerativeModel( + 'gemini-1.5-flash', + generation_config={ + "response_mime_type": "application/json", + "response_schema": dynamic_listings_container + } + ) prompt = SYSTEM_MESSAGE + "\n" + USER_MESSAGE + data # Count input tokens using Gemini's method input_tokens = model.count_tokens(prompt) @@ -242,23 +246,25 @@ def format_data(data, DynamicListingsContainer, DynamicListingModel, selected_mo "output_tokens": usage_metadata.candidates_token_count } return completion.text, token_counts - + elif selected_model == "Llama3.1 8B": # Dynamically generate the system message based on the schema - sys_message = generate_system_message(DynamicListingModel) - # print(SYSTEM_MESSAGE) + sys_message = generate_system_message(dynamic_listing_model) # Point to the local server - client = OpenAI(base_url="http://localhost:1234/v1", api_key="lm-studio") + client = OpenAI( + base_url="http://localhost:1234/v1", + api_key="lm-studio" + ) completion = client.chat.completions.create( - model=LLAMA_MODEL_FULLNAME, #change this if needed (use a better model) + # change this if needed (use a better model) + model=LLAMA_MODEL_FULLNAME, messages=[ {"role": "system", "content": sys_message}, {"role": "user", "content": USER_MESSAGE + data} ], - temperature=0.7, - + temperature=0.7 ) # Extract the content from the response @@ -266,7 +272,7 @@ def format_data(data, DynamicListingsContainer, DynamicListingModel, selected_mo print(response_content) # Convert the content from JSON string to a Python dictionary parsed_response = json.loads(response_content) - + # Extract token usage token_counts = { "input_tokens": completion.usage.prompt_tokens, @@ -274,28 +280,27 @@ def format_data(data, DynamicListingsContainer, DynamicListingModel, selected_mo } return parsed_response, token_counts - elif selected_model== "Groq Llama3.1 70b": - + elif selected_model == "Groq Llama3.1 70b": + # Dynamically generate the system message based on the schema - sys_message = generate_system_message(DynamicListingModel) - # print(SYSTEM_MESSAGE) + sys_message = generate_system_message(dynamic_listing_model) # Point to the local server client = Groq(api_key=get_api_key("GROQ_API_KEY"),) completion = client.chat.completions.create( - messages=[ - {"role": "system","content": sys_message}, - {"role": "user","content": USER_MESSAGE + data} - ], - model=GROQ_LLAMA_MODEL_FULLNAME, - ) + messages=[ + {"role": "system", "content": sys_message}, + {"role": "user", "content": USER_MESSAGE + data} + ], + model=GROQ_LLAMA_MODEL_FULLNAME, + ) # Extract the content from the response response_content = completion.choices[0].message.content - + # Convert the content from JSON string to a Python dictionary parsed_response = json.loads(response_content) - + # completion.usage token_counts = { "input_tokens": completion.usage.prompt_tokens, @@ -307,20 +312,21 @@ def format_data(data, DynamicListingsContainer, DynamicListingModel, selected_mo raise ValueError(f"Unsupported model: {selected_model}") - def save_formatted_data(formatted_data, output_folder: str, json_file_name: str, excel_file_name: str): """Save formatted data as JSON and Excel in the specified output folder.""" os.makedirs(output_folder, exist_ok=True) - + # Parse the formatted data if it's a JSON string (from Gemini API) if isinstance(formatted_data, str): try: formatted_data_dict = json.loads(formatted_data) - except json.JSONDecodeError: - raise ValueError("The provided formatted data is a string but not valid JSON.") + except json.JSONDecodeError as e: + raise ValueError( + "The provided formatted data is a string but not valid JSON.") from e else: # Handle data from OpenAI or other sources - formatted_data_dict = formatted_data.dict() if hasattr(formatted_data, 'dict') else formatted_data + formatted_data_dict = formatted_data.dict() if hasattr( + formatted_data, 'dict') else formatted_data # Save the formatted data as JSON json_output_path = os.path.join(output_folder, json_file_name) @@ -331,11 +337,13 @@ def save_formatted_data(formatted_data, output_folder: str, json_file_name: str, # Prepare data for DataFrame if isinstance(formatted_data_dict, dict): # If the data is a dictionary containing lists, assume these lists are records - data_for_df = next(iter(formatted_data_dict.values())) if len(formatted_data_dict) == 1 else formatted_data_dict + data_for_df = next(iter(formatted_data_dict.values())) if len( + formatted_data_dict) == 1 else formatted_data_dict elif isinstance(formatted_data_dict, list): data_for_df = formatted_data_dict else: - raise ValueError("Formatted data is neither a dictionary nor a list, cannot convert to DataFrame") + raise ValueError( + "Formatted data is neither a dictionary nor a list, cannot convert to DataFrame") # Create DataFrame try: @@ -346,27 +354,31 @@ def save_formatted_data(formatted_data, output_folder: str, json_file_name: str, excel_output_path = os.path.join(output_folder, excel_file_name) df.to_excel(excel_output_path, index=False) print(f"Formatted data saved to Excel at {excel_output_path}") - + return df + except Exception as e: print(f"Error creating DataFrame or saving Excel: {str(e)}") return None + def calculate_price(token_counts, model): input_token_count = token_counts.get("input_tokens", 0) output_token_count = token_counts.get("output_tokens", 0) - + # Calculate the costs input_cost = input_token_count * PRICING[model]["input"] output_cost = output_token_count * PRICING[model]["output"] total_cost = input_cost + output_cost - + return input_token_count, output_token_count, total_cost def generate_unique_folder_name(url): + """Generate a unique folder name based on the URL and current timestamp.""" timestamp = datetime.now().strftime('%Y_%m_%d__%H_%M_%S') - url_name = re.sub(r'\W+', '_', url.split('//')[1].split('/')[0]) # Extract domain name and replace non-alphanumeric characters + # Extract domain name and replace non-alphanumeric characters + url_name = re.sub(r'\W+', '_', url.split('//')[1].split('/')[0]) return f"{url_name}_{timestamp}" @@ -377,19 +389,33 @@ def scrape_url(url: str, fields: List[str], selected_model: str, output_folder: save_raw_data(markdown, output_folder, f'rawData_{file_number}.md') # Create the dynamic listing model - DynamicListingModel = create_dynamic_listing_model(fields) + dynamic_listing_model = create_dynamic_listing_model(fields) # Create the container model that holds a list of the dynamic listing models - DynamicListingsContainer = create_listings_container_model(DynamicListingModel) - + dynamic_listings_container = create_listings_container_model( + dynamic_listing_model) + # Format data - formatted_data, token_counts = format_data(markdown, DynamicListingsContainer, DynamicListingModel, selected_model) - + formatted_data, token_counts = format_data( + markdown, + dynamic_listings_container, + dynamic_listing_model, + selected_model + ) + # Save formatted data - save_formatted_data(formatted_data, output_folder, f'sorted_data_{file_number}.json', f'sorted_data_{file_number}.xlsx') + save_formatted_data( + formatted_data, + output_folder, + f'sorted_data_{file_number}.json', + f'sorted_data_{file_number}.xlsx' + ) # Calculate and return token usage and cost - input_tokens, output_tokens, total_cost = calculate_price(token_counts, selected_model) + input_tokens, output_tokens, total_cost = calculate_price( + token_counts, + selected_model + ) return input_tokens, output_tokens, total_cost, formatted_data except Exception as e: @@ -397,4 +423,3 @@ def scrape_url(url: str, fields: List[str], selected_model: str, output_folder: return 0, 0, 0, None # Remove the main execution block if it's not needed for testing purposes - \ No newline at end of file diff --git a/streamlit_app.py b/streamlit_app.py index 05b549dc..6fd40201 100644 --- a/streamlit_app.py +++ b/streamlit_app.py @@ -1,28 +1,44 @@ -# streamlit_app.py +#!/usr/bin/env python3 + +import os +import re +import json +from urllib.parse import urlparse +from datetime import datetime import streamlit as st from streamlit_tags import st_tags_sidebar import pandas as pd -import json -from datetime import datetime + from scraper import ( - fetch_html_selenium, - save_raw_data, - format_data, - save_formatted_data, calculate_price, - html_to_markdown_with_readability, create_dynamic_listing_model, create_listings_container_model, - scrape_url, - setup_selenium, - generate_unique_folder_name + fetch_html_selenium, + format_data, + html_to_markdown_with_readability, + save_formatted_data, + save_raw_data, + setup_selenium ) from pagination_detector import detect_pagination_elements -import re -from urllib.parse import urlparse from assets import PRICING -import os + + +def generate_unique_folder_name(source_url): + """Helper function to generate unique folder names based on the URL.""" + timestamp = datetime.now().strftime('%Y_%m_%d__%H_%M_%S') + # Parse the URL + parsed_url = urlparse(source_url) + # Extract the domain name + domain = parsed_url.netloc or parsed_url.path.split('/')[0] + # Remove 'www.' if present + domain = re.sub(r'^www\.', '', domain) + # Remove any non-alphanumeric characters and replace with underscores + clean_domain = re.sub(r'\W+', '_', domain) + + return f"{clean_domain}_{timestamp}" + # Initialize Streamlit app st.set_page_config(page_title="Universal Web Scraper", page_icon="🦑") @@ -30,7 +46,8 @@ # Initialize session state variables if 'scraping_state' not in st.session_state: - st.session_state['scraping_state'] = 'idle' # Possible states: 'idle', 'waiting', 'scraping', 'completed' + # Possible states: 'idle', 'waiting', 'scraping', 'completed' + st.session_state['scraping_state'] = 'idle' if 'results' not in st.session_state: st.session_state['results'] = None if 'driver' not in st.session_state: @@ -41,12 +58,19 @@ # API Keys with st.sidebar.expander("API Keys", expanded=False): - st.session_state['openai_api_key'] = st.text_input("OpenAI API Key", type="password") - st.session_state['gemini_api_key'] = st.text_input("Gemini API Key", type="password") - st.session_state['groq_api_key'] = st.text_input("Groq API Key", type="password") + st.session_state['openai_api_key'] = st.text_input( + "OpenAI API Key", type="password") + st.session_state['gemini_api_key'] = st.text_input( + "Gemini API Key", type="password") + st.session_state['groq_api_key'] = st.text_input( + "Groq API Key", type="password") # Model selection -model_selection = st.sidebar.selectbox("Select Model", options=list(PRICING.keys()), index=0) +model_selection = st.sidebar.selectbox( + "Select Model", + options=list(PRICING.keys()), + index=0 +) # URL input url_input = st.sidebar.text_input("Enter URL(s) separated by whitespace") @@ -87,13 +111,12 @@ # Multiple URLs entered; disable Pagination and Attended Mode use_pagination = False attended_mode = False - pagination_details = "" - st.sidebar.info("Pagination and Attended Mode are disabled when multiple URLs are entered.") + pagination_details = "" + st.sidebar.info( + "Pagination and Attended Mode are disabled when multiple URLs are entered.") st.sidebar.markdown("---") - - # Main action button if st.sidebar.button("LAUNCH SCRAPER", type="primary"): if url_input.strip() == "": @@ -120,7 +143,8 @@ st.write("Navigate to the page you want to scrape.") st.write("When ready, click the 'Resume Scraping' button.") else: - st.write("Browser window is already open. Perform your actions and click 'Resume Scraping'.") + st.write( + "Browser window is already open. Perform your actions and click 'Resume Scraping'.") if st.button("Resume Scraping"): st.session_state['scraping_state'] = 'scraping' @@ -129,7 +153,10 @@ elif st.session_state['scraping_state'] == 'scraping': with st.spinner('Scraping in progress...'): # Perform scraping - output_folder = os.path.join('output', generate_unique_folder_name(st.session_state['urls'][0])) + output_folder = os.path.join( + 'output', + generate_unique_folder_name(st.session_state['urls'][0]) + ) os.makedirs(output_folder, exist_ok=True) total_input_tokens = 0 @@ -142,23 +169,31 @@ if st.session_state['attended_mode'] and driver is not None: # Attended mode: scrape the current page without navigating # Fetch HTML from the current page - raw_html = fetch_html_selenium(st.session_state['urls'][0], attended_mode=True, driver=driver) + raw_html = fetch_html_selenium( + st.session_state['urls'][0], + attended_mode=True, + driver=driver + ) markdown = html_to_markdown_with_readability(raw_html) - save_raw_data(markdown, output_folder, f'rawData_1.md') + save_raw_data(markdown, output_folder, 'rawData_1.md') - current_url = driver.current_url # Use the current URL for logging and saving purposes + # Use the current URL for logging and saving purposes + current_url = driver.current_url # Detect pagination if enabled if st.session_state['use_pagination']: pagination_data, token_counts, pagination_price = detect_pagination_elements( - current_url, st.session_state['pagination_details'], st.session_state['model_selection'], markdown + current_url, + st.session_state['pagination_details'], + st.session_state['model_selection'], + markdown ) # Check if pagination_data is a dict or a model with 'page_urls' attribute if isinstance(pagination_data, dict): page_urls = pagination_data.get("page_urls", []) else: page_urls = pagination_data.page_urls - + pagination_info = { "page_urls": page_urls, "token_counts": token_counts, @@ -167,18 +202,31 @@ # Scrape data if fields are specified if show_tags: # Create dynamic models - DynamicListingModel = create_dynamic_listing_model(st.session_state['fields']) - DynamicListingsContainer = create_listings_container_model(DynamicListingModel) + DynamicListingModel = create_dynamic_listing_model( + st.session_state['fields']) + DynamicListingsContainer = create_listings_container_model( + DynamicListingModel) # Format data formatted_data, token_counts = format_data( - markdown, DynamicListingsContainer, DynamicListingModel, st.session_state['model_selection'] + markdown, + DynamicListingsContainer, + DynamicListingModel, + st.session_state['model_selection'] + ) + input_tokens, output_tokens, cost = calculate_price( + token_counts, + st.session_state['model_selection'] ) - input_tokens, output_tokens, cost = calculate_price(token_counts, st.session_state['model_selection']) total_input_tokens += input_tokens total_output_tokens += output_tokens total_cost += cost # Save formatted data - df = save_formatted_data(formatted_data, output_folder, f'sorted_data_1.json', f'sorted_data_1.xlsx') + df = save_formatted_data( + formatted_data, + output_folder, + 'sorted_data_1.json', + 'sorted_data_1.xlsx' + ) all_data.append(formatted_data) else: # Non-attended mode or driver not available @@ -186,19 +234,26 @@ # Fetch HTML raw_html = fetch_html_selenium(url, attended_mode=False) markdown = html_to_markdown_with_readability(raw_html) - save_raw_data(markdown, output_folder, f'rawData_{i}.md') + save_raw_data( + markdown, + output_folder, + f'rawData_{i}.md' + ) # Detect pagination if enabled and only for the first URL if st.session_state['use_pagination'] and i == 1: pagination_data, token_counts, pagination_price = detect_pagination_elements( - url, st.session_state['pagination_details'], st.session_state['model_selection'], markdown + url, + st.session_state['pagination_details'], + st.session_state['model_selection'], + markdown ) # Check if pagination_data is a dict or a model with 'page_urls' attribute if isinstance(pagination_data, dict): page_urls = pagination_data.get("page_urls", []) else: page_urls = pagination_data.page_urls - + pagination_info = { "page_urls": page_urls, "token_counts": token_counts, @@ -207,18 +262,31 @@ # Scrape data if fields are specified if show_tags: # Create dynamic models - DynamicListingModel = create_dynamic_listing_model(st.session_state['fields']) - DynamicListingsContainer = create_listings_container_model(DynamicListingModel) + DynamicListingModel = create_dynamic_listing_model( + st.session_state['fields']) + DynamicListingsContainer = create_listings_container_model( + DynamicListingModel) # Format data formatted_data, token_counts = format_data( - markdown, DynamicListingsContainer, DynamicListingModel, st.session_state['model_selection'] + markdown, + DynamicListingsContainer, + DynamicListingModel, + st.session_state['model_selection'] + ) + input_tokens, output_tokens, cost = calculate_price( + token_counts, + st.session_state['model_selection'] ) - input_tokens, output_tokens, cost = calculate_price(token_counts, st.session_state['model_selection']) total_input_tokens += input_tokens total_output_tokens += output_tokens total_cost += cost # Save formatted data - df = save_formatted_data(formatted_data, output_folder, f'sorted_data_{i}.json', f'sorted_data_{i}.xlsx') + df = save_formatted_data( + formatted_data, + output_folder, + f'sorted_data_{i}.json', + f'sorted_data_{i}.xlsx' + ) all_data.append(formatted_data) # Clean up driver if used @@ -236,6 +304,7 @@ 'pagination_info': pagination_info } st.session_state['scraping_state'] = 'completed' + # Display results if st.session_state['scraping_state'] == 'completed' and st.session_state['results']: results = st.session_state['results'] @@ -251,7 +320,7 @@ st.subheader("Scraping Results") for i, data in enumerate(all_data, start=1): st.write(f"Data from URL {i}:") - + # Handle string data (convert to dict if it's JSON) if isinstance(data, str): try: @@ -259,7 +328,7 @@ except json.JSONDecodeError: st.error(f"Failed to parse data as JSON for URL {i}") continue - + if isinstance(data, dict): if 'listings' in data and isinstance(data['listings'], list): df = pd.DataFrame(data['listings']) @@ -282,13 +351,17 @@ st.sidebar.markdown("#### Token Usage") st.sidebar.markdown(f"*Input Tokens:* {total_input_tokens}") st.sidebar.markdown(f"*Output Tokens:* {total_output_tokens}") - st.sidebar.markdown(f"**Total Cost:** :green-background[**${total_cost:.4f}**]") + st.sidebar.markdown( + f"**Total Cost:** :green-background[**${total_cost:.4f}**]") # Download options st.subheader("Download Extracted Data") col1, col2 = st.columns(2) with col1: - json_data = json.dumps(all_data, default=lambda o: o.dict() if hasattr(o, 'dict') else str(o), indent=4) + json_data = json.dumps( + all_data, + default=lambda o: o.dict() if hasattr(o, 'dict') else str(o), indent=4 + ) st.download_button( "Download JSON", data=json_data, @@ -306,10 +379,12 @@ if isinstance(data, dict) and 'listings' in data: all_listings.extend(data['listings']) elif hasattr(data, 'listings'): - all_listings.extend([item.dict() for item in data.listings]) + all_listings.extend( + [item.dict() for item in data.listings] + ) else: all_listings.append(data) - + combined_df = pd.DataFrame(all_listings) st.download_button( "Download CSV", @@ -327,32 +402,48 @@ # Display token usage and cost using metrics st.sidebar.markdown("---") st.sidebar.markdown("### Pagination Details") - st.sidebar.markdown(f"**Number of Page URLs:** {len(pagination_info['page_urls'])}") + st.sidebar.markdown( + f"**Number of Page URLs:** {len(pagination_info['page_urls'])}") st.sidebar.markdown("#### Pagination Token Usage") - st.sidebar.markdown(f"*Input Tokens:* {pagination_info['token_counts']['input_tokens']}") - st.sidebar.markdown(f"*Output Tokens:* {pagination_info['token_counts']['output_tokens']}") - st.sidebar.markdown(f"**Pagination Cost:** :blue-background[**${pagination_info['price']:.4f}**]") - + st.sidebar.markdown( + f"*Input Tokens:* {pagination_info['token_counts']['input_tokens']}") + st.sidebar.markdown( + f"*Output Tokens:* {pagination_info['token_counts']['output_tokens']}") + st.sidebar.markdown( + f"**Pagination Cost:** :blue-background[**${pagination_info['price']:.4f}**]") # Display page URLs in a table st.write("**Page URLs:**") # Make URLs clickable - pagination_df = pd.DataFrame(pagination_info["page_urls"], columns=["Page URLs"]) - + pagination_df = pd.DataFrame( + pagination_info["page_urls"], + columns=["Page URLs"] + ) + st.dataframe( pagination_df, column_config={ "Page URLs": st.column_config.LinkColumn("Page URLs") - },use_container_width=True + }, + use_container_width=True ) # Download pagination URLs st.subheader("Download Pagination URLs") col1, col2 = st.columns(2) with col1: - st.download_button("Download Pagination CSV",data=pagination_df.to_csv(index=False),file_name="pagination_urls.csv") + st.download_button( + "Download Pagination CSV", + data=pagination_df.to_csv(index=False), + file_name="pagination_urls.csv" + ) with col2: - st.download_button("Download Pagination JSON",data=json.dumps(pagination_info['page_urls'], indent=4),file_name="pagination_urls.json") + st.download_button( + "Download Pagination JSON", + data=json.dumps(pagination_info['page_urls'], indent=4), + file_name="pagination_urls.json" + ) + # Reset scraping state if st.sidebar.button("Clear Results"): st.session_state['scraping_state'] = 'idle' @@ -361,27 +452,13 @@ # If both scraping and pagination were performed, show totals under the pagination table if show_tags and pagination_info: st.markdown("---") - total_input_tokens_combined = total_input_tokens + pagination_info['token_counts']['input_tokens'] - total_output_tokens_combined = total_output_tokens + pagination_info['token_counts']['output_tokens'] + total_input_tokens_combined = total_input_tokens + \ + pagination_info['token_counts']['input_tokens'] + total_output_tokens_combined = total_output_tokens + \ + pagination_info['token_counts']['output_tokens'] total_combined_cost = total_cost + pagination_info['price'] st.markdown("### Total Counts and Cost (Including Pagination)") st.markdown(f"**Total Input Tokens:** {total_input_tokens_combined}") st.markdown(f"**Total Output Tokens:** {total_output_tokens_combined}") - st.markdown(f"**Total Combined Cost:** :rainbow-background[**${total_combined_cost:.4f}**]") -# Helper function to generate unique folder names -def generate_unique_folder_name(url): - timestamp = datetime.now().strftime('%Y_%m_%d__%H_%M_%S') - - # Parse the URL - parsed_url = urlparse(url) - - # Extract the domain name - domain = parsed_url.netloc or parsed_url.path.split('/')[0] - - # Remove 'www.' if present - domain = re.sub(r'^www\.', '', domain) - - # Remove any non-alphanumeric characters and replace with underscores - clean_domain = re.sub(r'\W+', '_', domain) - - return f"{clean_domain}_{timestamp}" + st.markdown( + f"**Total Combined Cost:** :rainbow-background[**${total_combined_cost:.4f}**]") diff --git a/test.py b/test.py deleted file mode 100644 index e69de29b..00000000